source: ftputil/sync.py @ 1718:8bed138bc404

Last change on this file since 1718:8bed138bc404 was 1718:8bed138bc404, checked in by Stefan Schwarzer <sschwarzer@…>, 11 months ago
Don't inherit from `object` This is no longer needed because we're targeting Python 3 only now.
File size: 5.9 KB
Line 
1# Copyright (C) 2007-2018, Stefan Schwarzer <sschwarzer@sschwarzer.net>
2# and ftputil contributors (see `doc/contributors.txt`)
3# See the file LICENSE for licensing terms.
4
5"""
6Tools for syncing combinations of local and remote directories.
7
8*** WARNING: This is an unfinished in-development version!
9"""
10
11# Sync combinations:
12# - remote -> local (download)
13# - local -> remote (upload)
14# - remote -> remote
15# - local -> local (maybe implicitly possible due to design, but not targeted)
16
17import os
18import shutil
19
20from ftputil import FTPHost
21import ftputil.error
22
23__all__ = ["FTPHost", "LocalHost", "Syncer"]
24
25
26# Used for copying file objects; value is 64 KB.
27CHUNK_SIZE = 64 * 1024
28
29
30class LocalHost:
31    """
32    Provide an API for local directories and files so we can use the
33    same code as for `FTPHost` instances.
34    """
35
36    def open(self, path, mode):
37        """
38        Return a Python file object for file name `path`, opened in
39        mode `mode`.
40        """
41        # This is the built-in `open` function, not `os.open`!
42        return open(path, mode)
43
44    def time_shift(self):
45        """
46        Return the time shift value (see methods `set_time_shift`
47        and `time_shift` in class `FTPHost` for a definition). By
48        definition, the value is zero for local file systems.
49        """
50        return 0.0
51
52    def __getattr__(self, attr):
53        return getattr(os, attr)
54
55
56class Syncer:
57    """
58    Control synchronization between combinations of local and remote
59    directories and files.
60    """
61
62    def __init__(self, source, target):
63        """
64        Init the `FTPSyncer` instance.
65
66        Each of `source` and `target` is either an `FTPHost` or a
67        `LocalHost` object. The source and target directories, resp.
68        have to be set with the `chdir` command before passing them
69        in. The semantics is so that the items under the source
70        directory will show up under the target directory after the
71        synchronization (unless there's an error).
72        """
73        self._source = source
74        self._target = target
75
76    def _mkdir(self, target_dir):
77        """
78        Try to create the target directory `target_dir`. If it already
79        exists, don't do anything. If the directory is present but
80        it's actually a file, raise a `SyncError`.
81        """
82        #TODO Handle setting of target mtime according to source mtime
83        # (beware of rootdir anomalies; try to handle them as well).
84        #print "Making", target_dir
85        if self._target.path.isfile(target_dir):
86            raise ftputil.error.SyncError(
87                  "target dir '{0}' is actually a file".format(target_dir))
88        # Deliberately use an `isdir` test instead of `try/except`. The
89        #  latter approach might mask other errors we want to see, e. g.
90        #  insufficient permissions.
91        if not self._target.path.isdir(target_dir):
92            self._target.mkdir(target_dir)
93
94    def _sync_file(self, source_file, target_file):
95        #XXX This duplicates code from `FTPHost._copyfileobj`. Maybe
96        # implement the upload and download methods in terms of
97        # `_sync_file`, or maybe not?
98        #TODO Handle `IOError`s
99        #TODO Handle conditional copy
100        #TODO Handle setting of target mtime according to source mtime
101        # (beware of rootdir anomalies; try to handle them as well).
102        #print "Syncing", source_file, "->", target_file
103        source = self._source.open(source_file, "rb")
104        try:
105            target = self._target.open(target_file, "wb")
106            try:
107                shutil.copyfileobj(source, target, length=CHUNK_SIZE)
108            finally:
109                target.close()
110        finally:
111            source.close()
112
113    def _fix_sep_for_target(self, path):
114        """
115        Return the string `path` with appropriate path separators for
116        the target file system.
117        """
118        return path.replace(self._source.sep, self._target.sep)
119
120    def _sync_tree(self, source_dir, target_dir):
121        """
122        Synchronize the source and the target directory tree by
123        updating the target to match the source as far as possible.
124
125        Current limitations:
126        - _don't_ delete items which are on the target path but not on the
127          source path
128        - files are always copied, the modification timestamps are not
129          compared
130        - all files are copied in binary mode, never in ASCII/text mode
131        - incomplete error handling
132        """
133        self._mkdir(target_dir)
134        for dirpath, dir_names, file_names in self._source.walk(source_dir):
135            for dir_name in dir_names:
136                inner_source_dir = self._source.path.join(dirpath, dir_name)
137                inner_target_dir = inner_source_dir.replace(source_dir,
138                                                            target_dir, 1)
139                inner_target_dir = self._fix_sep_for_target(inner_target_dir)
140                self._mkdir(inner_target_dir)
141            for file_name in file_names:
142                source_file = self._source.path.join(dirpath, file_name)
143                target_file = source_file.replace(source_dir, target_dir, 1)
144                target_file = self._fix_sep_for_target(target_file)
145                self._sync_file(source_file, target_file)
146
147    def sync(self, source_path, target_path):
148        """
149        Synchronize `source_path` and `target_path` (both are strings,
150        each denoting a directory or file path), i. e. update the
151        target path so that it's a copy of the source path.
152
153        This method handles both directory trees and single files.
154        """
155        #TODO Handle making of missing intermediate directories
156        source_path = self._source.path.abspath(source_path)
157        target_path = self._target.path.abspath(target_path)
158        if self._source.path.isfile(source_path):
159            self._sync_file(source_path, target_path)
160        else:
161            self._sync_tree(source_path, target_path)
Note: See TracBrowser for help on using the repository browser.