source: ftputil/file_transfer.py @ 1580:95647a76b6c5

Last change on this file since 1580:95647a76b6c5 was 1580:95647a76b6c5, checked in by Stefan Schwarzer <sschwarzer@…>, 5 years ago
Be explicit about unknown datetime precisions. Instead of relying on default handling of pre-epoch times, explicitly consider unknown precisions and in this case, consider the source file newer. That is, if in doubt, transfer the file (as was the assumption before).
File size: 6.3 KB
Line 
1# Copyright (C) 2013-2014, Stefan Schwarzer <sschwarzer@sschwarzer.net>
2# and ftputil contributors (see `doc/contributors.txt`)
3# See the file LICENSE for licensing terms.
4
5"""
6file_transfer.py - upload, download and generic file copy
7"""
8
9from __future__ import unicode_literals
10
11import io
12import os
13
14import ftputil.stat
15
16
17#TODO Think a bit more about the API before making it public.
18# # Only `chunks` should be used by clients of the ftputil library. Any
19# #  other functionality is supposed to be used via `FTPHost` objects.
20# __all__ = ["chunks"]
21__all__ = []
22
23# Maximum size of chunk in `FTPHost.copyfileobj` in bytes.
24MAX_COPY_CHUNK_SIZE = 64 * 1024
25
26
27class LocalFile(object):
28    """
29    Represent a file on the local side which is to be transferred or
30    is already transferred.
31    """
32
33    def __init__(self, name, mode):
34        self.name = os.path.abspath(name)
35        self.mode = mode
36
37    def exists(self):
38        """
39        Return `True` if the path representing this file exists.
40        Otherwise return `False`.
41        """
42        return os.path.exists(self.name)
43
44    def mtime(self):
45        """Return the timestamp for the last modification in seconds."""
46        return os.path.getmtime(self.name)
47
48    def mtime_precision(self):
49        """Return the precision of the last modification time in seconds."""
50        # Derived classes might want to use `self`.
51        # pylint: disable=no-self-use
52        #
53        # Assume modification timestamps for local file systems are
54        # at least precise up to a second.
55        return 1.0
56
57    def fobj(self):
58        """Return a file object for the name/path in the constructor."""
59        return io.open(self.name, self.mode)
60
61
62class RemoteFile(object):
63    """
64    Represent a file on the remote side which is to be transferred or
65    is already transferred.
66    """
67
68    def __init__(self, ftp_host, name, mode):
69        self._host = ftp_host
70        self._path = ftp_host.path
71        self.name = self._path.abspath(name)
72        self.mode = mode
73
74    def exists(self):
75        """
76        Return `True` if the path representing this file exists.
77        Otherwise return `False`.
78        """
79        return self._path.exists(self.name)
80
81    def mtime(self):
82        """Return the timestamp for the last modification in seconds."""
83        # Convert to client time zone (see definition of time
84        # shift in docstring of `FTPHost.set_time_shift`).
85        return self._path.getmtime(self.name) - self._host.time_shift()
86
87    def mtime_precision(self):
88        """Return the precision of the last modification time in seconds."""
89        # I think using `stat` instead of `lstat` makes more sense here.
90        return self._host.stat(self.name)._st_mtime_precision
91
92    def fobj(self):
93        """Return a file object for the name/path in the constructor."""
94        return self._host.open(self.name, self.mode)
95
96
97def source_is_newer_than_target(source_file, target_file):
98    """
99    Return `True` if the source is newer than the target, else `False`.
100
101    Both arguments are `LocalFile` or `RemoteFile` objects.
102
103    It's assumed that the actual modification time is
104
105      reported_mtime <= actual_mtime <= reported_mtime + mtime_precision
106
107    i. e. that the reported mtime is the actual mtime or rounded down
108    (truncated).
109
110    For the purpose of this test the source is newer than the target
111    if any of the possible actual source modification times is greater
112    than the reported target modification time. In other words: If in
113    doubt, the file should be transferred.
114
115    This is the only situation where the source is _not_ considered
116    newer than the target:
117
118    |/////////////////////|              possible source mtime
119                            |////////|   possible target mtime
120
121    That is, the latest possible actual source modification time is
122    before the first possible actual target modification time.
123    """
124    if source_file.mtime_precision() is ftputil.stat.UNKNOWN_PRECISION:
125        return True
126    else:
127        return (source_file.mtime() + source_file.mtime_precision() >=
128                target_file.mtime())
129
130
131def chunks(fobj, max_chunk_size=MAX_COPY_CHUNK_SIZE):
132    """
133    Return an iterator which yields the contents of the file object.
134
135    For each iteration, at most `max_chunk_size` bytes are read from
136    `fobj` and yielded as a byte string. If the file object is
137    exhausted, then don't yield any more data but stop the iteration,
138    so the client does _not_ get an empty byte string.
139
140    Any exceptions resulting from reading the file object are passed
141    through to the client.
142    """
143    while True:
144        chunk = fobj.read(max_chunk_size)
145        if not chunk:
146            break
147        yield chunk
148
149
150def copyfileobj(source_fobj, target_fobj, max_chunk_size=MAX_COPY_CHUNK_SIZE,
151                callback=None):
152    """Copy data from file-like object source to file-like object target."""
153    # Inspired by `shutil.copyfileobj` (I don't use the `shutil`
154    # code directly because it might change)
155    for chunk in chunks(source_fobj, max_chunk_size):
156        target_fobj.write(chunk)
157        if callback is not None:
158            callback(chunk)
159
160
161def copy_file(source_file, target_file, conditional, callback):
162    """
163    Copy a file from `source_file` to `target_file`.
164
165    These are `LocalFile` or `RemoteFile` objects. Which of them
166    is a local or a remote file, respectively, is determined by
167    the arguments. If `conditional` is true, the file is only
168    copied if the target doesn't exist or is older than the
169    source. If `conditional` is false, the file is copied
170    unconditionally. Return `True` if the file was copied, else
171    `False`.
172    """
173    if conditional:
174        # Evaluate condition: The target file either doesn't exist or is
175        # older than the source file. If in doubt (due to imprecise
176        # timestamps), perform the transfer.
177        transfer_condition = not target_file.exists() or \
178          source_is_newer_than_target(source_file, target_file)
179        if not transfer_condition:
180            # We didn't transfer.
181            return False
182    source_fobj = source_file.fobj()
183    try:
184        target_fobj = target_file.fobj()
185        try:
186            copyfileobj(source_fobj, target_fobj, callback=callback)
187        finally:
188            target_fobj.close()
189    finally:
190        source_fobj.close()
191    # Transfer accomplished
192    return True
Note: See TracBrowser for help on using the repository browser.