Changeset 548:755b538cb146

Timestamp:

2006-10-14 01:48:43 (4 years ago)

Author:

Stefan Schwarzer <sschwarzer@…>

Branch:

add_stat_caching

convert_revision:

svn:778c30c8-61e0-0310-89d4-fe2f97a467b2/branches/add_stat_caching@571

Message:

Let ftputil use the stat cache in ftp_stat_cache.py. With the script
sandbox/list_dir_test.py, this gives a reduction of the running time
to about a tenth!

Warning: The code isn't production-ready yet. For example, the cache
code isn't thouroughly tested and has no expiration strategy at all.

Files:

• _test_ftp_stat.py (modified) (1 diff)
• _test_ftputil.py (modified) (1 diff)
• ftp_stat.py (modified) (7 diffs)
• ftp_stat_cache.py (modified) (1 diff)
• ftputil.py (modified) (4 diffs)

_test_ftp_stat.py

@@ -325 +325 @@
         """Test non-switching of parser format with `PermanentError`."""
         self.assertEqual(self.stat._allow_parser_switching, True)
-        # if there's a `PermanentError`, don't switch because we
-        #  don't know if the file was missed due to a wrong parser
+        # with these directory contents, we get a `ParserError` for
+        #  the Unix parser, so `_allow_parser_switching` can be
+        #  switched off no matter whether we got a `PermanentError`
+        #  or not
         self.assertRaises(ftp_error.PermanentError, self.stat.lstat,
                           "/home/msformat/nonexistent")
-        self.assertEqual(self.stat._allow_parser_switching, True)
+        self.assertEqual(self.stat._allow_parser_switching, False)
 
     def test_parser_switching_default_to_unix(self):

_test_ftputil.py

@@ -122 +122 @@
         def getmtime(self, file_name):
             return self._mtime
+        def abspath(self, path):
+            return "/home/sschwarzer/_ftputil_sync_"
         # needed for `isdir` in `FTPHost.remove`
         def isfile(self, path):

ftp_stat.py

@@ -41 +41 @@
 
 import ftp_error
+import ftp_stat_cache
 
 
@@ -292 +293 @@
         #  doesn't work
         self._allow_parser_switching = True
+        # cache only lstat results; `stat` works locally on `lstat` results
+        self._lstat_cache = ftp_stat_cache.StatCache()
 
     def _host_dir(self, path):
@@ -331 +334 @@
         return names
 
-    def _stat_candidates(self, lines, wanted_name):
-        """Return candidate lines for further analysis."""
-        # return only lines that contain the name of the file to stat
-        #  (however, the string may be _anywhere_ on the line but not
-        #  necessarily the file's basename; e. g. the string could
-        #  occur as the name of the file's group)
-        return [line  for line in lines  if wanted_name in line]
-
     def _real_lstat(self, path, _exception_for_missing_path=True):
         """
@@ -353 +348 @@
         _not_ intended for use by ftputil clients.)
         """
+        path = self._path.abspath(path)
+        # if the path is in the cache, return the lstat result
+        if path in self._lstat_cache:
+            return self._lstat_cache[path]
         # get output from FTP's `DIR` command
         lines = []
-        path = self._path.abspath(path)
         # Note: (l)stat works by going one directory up and parsing
         #  the output of an FTP `DIR` command. Unfortunately, it is
@@ -361 +359 @@
         if path == '/':
             raise ftp_error.RootDirError(
-                  "can't invoke stat for remote root directory")
+                  "can't stat remote root directory")
         dirname, basename = self._path.split(path)
+        lstat_result_for_path = None
+        # loop through all lines of the directory listing; we
+        #  probably won't need all lines for the particular path but
+        #  we want to collect as many stat results in the cache as
+        #  possible
         lines = self._host_dir(dirname)
-        # search for name to be stat'ed without parsing the whole
-        #  directory listing
-        candidates = self._stat_candidates(lines, basename)
-        # parse candidates; return the first stat result where the
-        #  calculated name matches the previously determined
-        #  basename
-        for line in candidates:
+        for line in lines:
             try:
                 stat_result = self._parser.parse_line(line,
                               self._host.time_shift())
+                loop_path = self._path.join(dirname, stat_result._st_name)
+                self._lstat_cache[loop_path] = stat_result
+                # needed to work without or with disabled cache
                 if stat_result._st_name == basename:
-                    return stat_result
+                    lstat_result_for_path = stat_result
             except ftp_error.ParserError:
                 # ignore things like "total 17", as found in some
@@ -381 +381 @@
                 if not line.lower().startswith("total"):
                     raise
+        if lstat_result_for_path:
+            return lstat_result_for_path
         # path was not found
         if _exception_for_missing_path:
@@ -426 +428 @@
             path = self._path.normpath(path)
             # check for cyclic structure
-            if visited_paths.has_key(path):
+            if path in visited_paths:
                 # we had this path already
                 raise ftp_error.PermanentError(

ftp_stat_cache.py

@@ -45 +45 @@
         self._cache = {}
         self._debug = False
-        self.enabled = False
+        self.enabled = True
 
     def _debug_output(self, text):

ftputil.py

@@ -138 +138 @@
         # lstat, stat, listdir services
         self._stat = ftp_stat._Stat(self)
+        self.stat_cache = self._stat._lstat_cache
         # save (cache) current directory
         self._current_dir = ftp_error._try_with_oserror(self._session.pwd)
@@ -499 +500 @@
                                                # use "old" current dir
                                                self._current_dir, path))
-        #self._current_dir = ftp_error._try_with_oserror(self._session.pwd)
 
     def mkdir(self, path, mode=None):
@@ -542 +542 @@
         is no longer supported.
         """
+        path = self.path.abspath(path)
         if self.listdir(path):
-            path = self.path.abspath(path)
             raise ftp_error.PermanentError("directory '%s' not empty" % path)
         #XXX how will `rmd` work with links?
         ftp_error._try_with_oserror(self._session.rmd, path)
+        self.stat_cache.invalidate(path)
 
     def remove(self, path):
         """Remove the given file or link."""
+        path = self.path.abspath(path)
         # though `isfile` includes also links to files, `islink`
         #  is needed to include links to directories
@@ -557 +559 @@
             raise ftp_error.PermanentError("remove/unlink can only delete "
                                            "files and links, not directories")
+        self.stat_cache.invalidate(path)
 
     def unlink(self, path):