source: ftputil/file_transfer.py @ 1713:f146a1ea66aa

Last change on this file since 1713:f146a1ea66aa was 1713:f146a1ea66aa, checked in by Stefan Schwarzer <sschwarzer@…>, 4 months ago
Remove `__future__` imports With the switch to Python 3.x-only, the `__future__` imports are no longer needed. Update copyright years along with the `__future__` import removal.
File size: 6.3 KB
Line 
1# Copyright (C) 2013-2018, Stefan Schwarzer <sschwarzer@sschwarzer.net>
2# and ftputil contributors (see `doc/contributors.txt`)
3# See the file LICENSE for licensing terms.
4
5"""
6file_transfer.py - upload, download and generic file copy
7"""
8
9import io
10import os
11
12import ftputil.stat
13
14
15#TODO Think a bit more about the API before making it public.
16# # Only `chunks` should be used by clients of the ftputil library. Any
17# #  other functionality is supposed to be used via `FTPHost` objects.
18# __all__ = ["chunks"]
19__all__ = []
20
21# Maximum size of chunk in `FTPHost.copyfileobj` in bytes.
22MAX_COPY_CHUNK_SIZE = 64 * 1024
23
24
25class LocalFile(object):
26    """
27    Represent a file on the local side which is to be transferred or
28    is already transferred.
29    """
30
31    def __init__(self, name, mode):
32        self.name = os.path.abspath(name)
33        self.mode = mode
34
35    def exists(self):
36        """
37        Return `True` if the path representing this file exists.
38        Otherwise return `False`.
39        """
40        return os.path.exists(self.name)
41
42    def mtime(self):
43        """Return the timestamp for the last modification in seconds."""
44        return os.path.getmtime(self.name)
45
46    def mtime_precision(self):
47        """Return the precision of the last modification time in seconds."""
48        # Derived classes might want to use `self`.
49        # pylint: disable=no-self-use
50        #
51        # Assume modification timestamps for local file systems are
52        # at least precise up to a second.
53        return 1.0
54
55    def fobj(self):
56        """Return a file object for the name/path in the constructor."""
57        return io.open(self.name, self.mode)
58
59
60class RemoteFile(object):
61    """
62    Represent a file on the remote side which is to be transferred or
63    is already transferred.
64    """
65
66    def __init__(self, ftp_host, name, mode):
67        self._host = ftp_host
68        self._path = ftp_host.path
69        self.name = self._path.abspath(name)
70        self.mode = mode
71
72    def exists(self):
73        """
74        Return `True` if the path representing this file exists.
75        Otherwise return `False`.
76        """
77        return self._path.exists(self.name)
78
79    def mtime(self):
80        """Return the timestamp for the last modification in seconds."""
81        # Convert to client time zone (see definition of time
82        # shift in docstring of `FTPHost.set_time_shift`).
83        return self._path.getmtime(self.name) - self._host.time_shift()
84
85    def mtime_precision(self):
86        """Return the precision of the last modification time in seconds."""
87        # I think using `stat` instead of `lstat` makes more sense here.
88        return self._host.stat(self.name)._st_mtime_precision
89
90    def fobj(self):
91        """Return a file object for the name/path in the constructor."""
92        return self._host.open(self.name, self.mode)
93
94
95def source_is_newer_than_target(source_file, target_file):
96    """
97    Return `True` if the source is newer than the target, else `False`.
98
99    Both arguments are `LocalFile` or `RemoteFile` objects.
100
101    It's assumed that the actual modification time is
102
103      reported_mtime <= actual_mtime <= reported_mtime + mtime_precision
104
105    i. e. that the reported mtime is the actual mtime or rounded down
106    (truncated).
107
108    For the purpose of this test the source is newer than the target
109    if any of the possible actual source modification times is greater
110    than the reported target modification time. In other words: If in
111    doubt, the file should be transferred.
112
113    This is the only situation where the source is _not_ considered
114    newer than the target:
115
116    |/////////////////////|              possible source mtime
117                            |////////|   possible target mtime
118
119    That is, the latest possible actual source modification time is
120    before the first possible actual target modification time.
121    """
122    if source_file.mtime_precision() is ftputil.stat.UNKNOWN_PRECISION:
123        return True
124    else:
125        return (source_file.mtime() + source_file.mtime_precision() >=
126                target_file.mtime())
127
128
129def chunks(fobj, max_chunk_size=MAX_COPY_CHUNK_SIZE):
130    """
131    Return an iterator which yields the contents of the file object.
132
133    For each iteration, at most `max_chunk_size` bytes are read from
134    `fobj` and yielded as a byte string. If the file object is
135    exhausted, then don't yield any more data but stop the iteration,
136    so the client does _not_ get an empty byte string.
137
138    Any exceptions resulting from reading the file object are passed
139    through to the client.
140    """
141    while True:
142        chunk = fobj.read(max_chunk_size)
143        if not chunk:
144            break
145        yield chunk
146
147
148def copyfileobj(source_fobj, target_fobj, max_chunk_size=MAX_COPY_CHUNK_SIZE,
149                callback=None):
150    """Copy data from file-like object source to file-like object target."""
151    # Inspired by `shutil.copyfileobj` (I don't use the `shutil`
152    # code directly because it might change)
153    for chunk in chunks(source_fobj, max_chunk_size):
154        target_fobj.write(chunk)
155        if callback is not None:
156            callback(chunk)
157
158
159def copy_file(source_file, target_file, conditional, callback):
160    """
161    Copy a file from `source_file` to `target_file`.
162
163    These are `LocalFile` or `RemoteFile` objects. Which of them
164    is a local or a remote file, respectively, is determined by
165    the arguments. If `conditional` is true, the file is only
166    copied if the target doesn't exist or is older than the
167    source. If `conditional` is false, the file is copied
168    unconditionally. Return `True` if the file was copied, else
169    `False`.
170    """
171    if conditional:
172        # Evaluate condition: The target file either doesn't exist or is
173        # older than the source file. If in doubt (due to imprecise
174        # timestamps), perform the transfer.
175        transfer_condition = not target_file.exists() or \
176          source_is_newer_than_target(source_file, target_file)
177        if not transfer_condition:
178            # We didn't transfer.
179            return False
180    source_fobj = source_file.fobj()
181    try:
182        target_fobj = target_file.fobj()
183        try:
184            copyfileobj(source_fobj, target_fobj, callback=callback)
185        finally:
186            target_fobj.close()
187    finally:
188        source_fobj.close()
189    # Transfer accomplished
190    return True
Note: See TracBrowser for help on using the repository browser.