source: ftputil/sync.py @ 1712:1b17d07f3a88

Last change on this file since 1712:1b17d07f3a88 was 1564:c5b353a1c23d, checked in by Stefan Schwarzer <sschwarzer@…>, 5 years ago
List contributors in `doc/contributors.txt`. So far, individual files had copyright notices for contributors. However, this makes it difficult to properly adapt files in case of refactoring: If a piece of code is moved to another file, I would need to find out if this code was contributed by someone else and change the copyright notice in the target file accordingly. With the new approach, every file refers to the file `doc/contributors.txt`, which contains the names of contributors.
File size: 5.9 KB
Line 
1# Copyright (C) 2007-2012, Stefan Schwarzer <sschwarzer@sschwarzer.net>
2# and ftputil contributors (see `doc/contributors.txt`)
3# See the file LICENSE for licensing terms.
4
5"""
6Tools for syncing combinations of local and remote directories.
7
8*** WARNING: This is an unfinished in-development version!
9"""
10
11# Sync combinations:
12# - remote -> local (download)
13# - local -> remote (upload)
14# - remote -> remote
15# - local -> local (maybe implicitly possible due to design, but not targeted)
16
17from __future__ import unicode_literals
18
19import os
20import shutil
21
22from ftputil import FTPHost
23import ftputil.error
24
25__all__ = ["FTPHost", "LocalHost", "Syncer"]
26
27
28# Used for copying file objects; value is 64 KB.
29CHUNK_SIZE = 64 * 1024
30
31
32class LocalHost(object):
33    """
34    Provide an API for local directories and files so we can use the
35    same code as for `FTPHost` instances.
36    """
37
38    def open(self, path, mode):
39        """
40        Return a Python file object for file name `path`, opened in
41        mode `mode`.
42        """
43        # This is the built-in `open` function, not `os.open`!
44        return open(path, mode)
45
46    def time_shift(self):
47        """
48        Return the time shift value (see methods `set_time_shift`
49        and `time_shift` in class `FTPHost` for a definition). By
50        definition, the value is zero for local file systems.
51        """
52        return 0.0
53
54    def __getattr__(self, attr):
55        return getattr(os, attr)
56
57
58class Syncer(object):
59    """
60    Control synchronization between combinations of local and remote
61    directories and files.
62    """
63
64    def __init__(self, source, target):
65        """
66        Init the `FTPSyncer` instance.
67
68        Each of `source` and `target` is either an `FTPHost` or a
69        `LocalHost` object. The source and target directories, resp.
70        have to be set with the `chdir` command before passing them
71        in. The semantics is so that the items under the source
72        directory will show up under the target directory after the
73        synchronization (unless there's an error).
74        """
75        self._source = source
76        self._target = target
77
78    def _mkdir(self, target_dir):
79        """
80        Try to create the target directory `target_dir`. If it already
81        exists, don't do anything. If the directory is present but
82        it's actually a file, raise a `SyncError`.
83        """
84        #TODO Handle setting of target mtime according to source mtime
85        # (beware of rootdir anomalies; try to handle them as well).
86        #print "Making", target_dir
87        if self._target.path.isfile(target_dir):
88            raise ftputil.error.SyncError(
89                  "target dir '{0}' is actually a file".format(target_dir))
90        # Deliberately use an `isdir` test instead of `try/except`. The
91        #  latter approach might mask other errors we want to see, e. g.
92        #  insufficient permissions.
93        if not self._target.path.isdir(target_dir):
94            self._target.mkdir(target_dir)
95
96    def _sync_file(self, source_file, target_file):
97        #XXX This duplicates code from `FTPHost._copyfileobj`. Maybe
98        # implement the upload and download methods in terms of
99        # `_sync_file`, or maybe not?
100        #TODO Handle `IOError`s
101        #TODO Handle conditional copy
102        #TODO Handle setting of target mtime according to source mtime
103        # (beware of rootdir anomalies; try to handle them as well).
104        #print "Syncing", source_file, "->", target_file
105        source = self._source.open(source_file, "rb")
106        try:
107            target = self._target.open(target_file, "wb")
108            try:
109                shutil.copyfileobj(source, target, length=CHUNK_SIZE)
110            finally:
111                target.close()
112        finally:
113            source.close()
114
115    def _fix_sep_for_target(self, path):
116        """
117        Return the string `path` with appropriate path separators for
118        the target file system.
119        """
120        return path.replace(self._source.sep, self._target.sep)
121
122    def _sync_tree(self, source_dir, target_dir):
123        """
124        Synchronize the source and the target directory tree by
125        updating the target to match the source as far as possible.
126
127        Current limitations:
128        - _don't_ delete items which are on the target path but not on the
129          source path
130        - files are always copied, the modification timestamps are not
131          compared
132        - all files are copied in binary mode, never in ASCII/text mode
133        - incomplete error handling
134        """
135        self._mkdir(target_dir)
136        for dirpath, dir_names, file_names in self._source.walk(source_dir):
137            for dir_name in dir_names:
138                inner_source_dir = self._source.path.join(dirpath, dir_name)
139                inner_target_dir = inner_source_dir.replace(source_dir,
140                                                            target_dir, 1)
141                inner_target_dir = self._fix_sep_for_target(inner_target_dir)
142                self._mkdir(inner_target_dir)
143            for file_name in file_names:
144                source_file = self._source.path.join(dirpath, file_name)
145                target_file = source_file.replace(source_dir, target_dir, 1)
146                target_file = self._fix_sep_for_target(target_file)
147                self._sync_file(source_file, target_file)
148
149    def sync(self, source_path, target_path):
150        """
151        Synchronize `source_path` and `target_path` (both are strings,
152        each denoting a directory or file path), i. e. update the
153        target path so that it's a copy of the source path.
154
155        This method handles both directory trees and single files.
156        """
157        #TODO Handle making of missing intermediate directories
158        source_path = self._source.path.abspath(source_path)
159        target_path = self._target.path.abspath(target_path)
160        if self._source.path.isfile(source_path):
161            self._sync_file(source_path, target_path)
162        else:
163            self._sync_tree(source_path, target_path)
Note: See TracBrowser for help on using the repository browser.