root/branches/add_stat_caching/ftpsync-0.1/caching_ftp.py

Revision 545, 3.9 kB (checked in by schwa, 2 years ago) 
Code contributed by Martin Wilck. (Thank you!) I might use some of
this code to add caching of stat results to ftputil.
  • Property svn:eol-style set to native
Line 
1 from ftputil import FTPHost
2 from ftputil.ftp_error import PermanentError, InternalError
3 from loggingclass import LoggingClass
4 from simplecache import Cache
5 from casepath import CaseInsPath, CaseInsStat
6
7 class CachingFTPHost(FTPHost, LoggingClass):
8
9     """
10     This class is like ftputil.FTPHost, except that
11     the working directory and directory contents are cached.
12     This may speed up FTP operations significantly, especially
13     when traversing trees and looking for stat() like information.
14
15     However, cached information may be wrong in some cases.
16     It is recommended to call host.invalidate_dir(<path>) after
17     closing the ftp_file object associated with <path>.
18
19     Constructor keywords "expire" and "size" are the same as for
20     simplecache.Cache.
21
22     Besides, this class adds a method check_case_insensitive() to cope
23     with FTP servers that don't distinguish file names by case.
24
25     """
26
27     def __init__(self, *args, **kwargs):
28
29         kw = {}
30         if "expire" in kwargs:
31             kw["expire"] = kwargs["expire"]
32             del kwargs["expire"]
33         if "size" in kwargs:
34             kw["size"] = kwargs["size"]
35             del kwargs["size"]
36         self.cache = Cache(**kw)
37
38         FTPHost.__init__(self, *args, **kwargs)
39         self.CWD = None
40         self.setcwd()
41
42     def check_case_insensitive(self):
43         """
44         Check for server case-insensivity and
45         This function must be called with an established connection and
46         with write permissions (like synchronize_times).
47         
48         """
49         helper_name = "__CachingFtpHost_Helper__"
50         try:
51             self.mkdir(helper_name)
52             self.lstat(helper_name) # exception if mkdir failed
53             try:
54                 # if the server is case-insensitive, this will fail
55                 file = self.mkdir(helper_name.lower())
56             except PermanentError:
57                 self.logger.warning("Server is case-insensitive")
58                 self.path = CaseInsPath(self)
59                 self._stat = CaseInsStat(self)
60                 self.cache.invalidate_all()
61             else:
62                 self.logger.info("Server is case-sensitive")
63         finally:
64             self.rmdir(helper_name)
65
66     def getcwd(self):
67         """
68         Return cached working directory.
69         """
70         return self.CWD
71
72     def setcwd(self):
73         """
74         Update cached working directory.
75         """
76         self.CWD = self.path.normpath(FTPHost.getcwd(self))
77         self.logger.info("New cwd: %s" % self.CWD)
78
79     def chdir(self, path):
80         FTPHost.chdir(self, path)
81         self.setcwd()
82
83     def _dir(self, path):
84
85         path = self.path.normcase(path)
86         try:
87             lines = self.cache[path]
88         except KeyError:
89             self.logger.debug("cache miss: %s" % path)
90             lines = FTPHost._dir(self, path)
91             self.cache[path] = lines
92         else:
93             self.logger.debug("cache hit: %s" % path)
94         return lines
95
96     def _invalidate_dir(self, path):
97         self.logger.debug("invalidating cache for %s" % path)
98         self.cache.invalidate(
99             self.path.normcase(
100             self.path.dirname(self.path.abspath(path))))
101
102     def file(self, path, mode='r'):
103         path = self.path.abspath(path)
104         ret = FTPHost.file(self, path, mode)
105         if 'w' in mode:
106             self._invalidate_dir(path)
107         return ret
108
109     def mkdir(self, path, mode=None):
110         path = self.path.abspath(path)
111         FTPHost.mkdir(self, path, mode)
112         self._invalidate_dir(path)
113
114     def rmdir(self, path):
115         FTPHost.rmdir(self, path)
116         self.cache.invalidate(self.path.normcase(
117             self.path.abspath(path)))
118         self._invalidate_dir(path)
119
120     def remove(self, path):
121         FTPHost.remove(self, path)
122         self._invalidate_dir(path)
123
124     def rename(self, source, target):
125         FTPHost.rename(self, source, target)
126         self._invalidate_dir(source)
127         self._invalidate_dir(target)
Note: See TracBrowser for help on using the browser.