source: ftputil/file_transfer.py @ 1717:827cfaff87d7

Last change on this file since 1717:827cfaff87d7 was 1717:827cfaff87d7, checked in by Stefan Schwarzer <sschwarzer@…>, 7 months ago
Remove unneeded uses of `io` In Python 3, `io.open` is the same as `open`, so we don't need to use `io.open`.
File size: 6.2 KB
Line 
1# Copyright (C) 2013-2018, Stefan Schwarzer <sschwarzer@sschwarzer.net>
2# and ftputil contributors (see `doc/contributors.txt`)
3# See the file LICENSE for licensing terms.
4
5"""
6file_transfer.py - upload, download and generic file copy
7"""
8
9import os
10
11import ftputil.stat
12
13
14#TODO Think a bit more about the API before making it public.
15# # Only `chunks` should be used by clients of the ftputil library. Any
16# #  other functionality is supposed to be used via `FTPHost` objects.
17# __all__ = ["chunks"]
18__all__ = []
19
20# Maximum size of chunk in `FTPHost.copyfileobj` in bytes.
21MAX_COPY_CHUNK_SIZE = 64 * 1024
22
23
24class LocalFile(object):
25    """
26    Represent a file on the local side which is to be transferred or
27    is already transferred.
28    """
29
30    def __init__(self, name, mode):
31        self.name = os.path.abspath(name)
32        self.mode = mode
33
34    def exists(self):
35        """
36        Return `True` if the path representing this file exists.
37        Otherwise return `False`.
38        """
39        return os.path.exists(self.name)
40
41    def mtime(self):
42        """Return the timestamp for the last modification in seconds."""
43        return os.path.getmtime(self.name)
44
45    def mtime_precision(self):
46        """Return the precision of the last modification time in seconds."""
47        # Derived classes might want to use `self`.
48        # pylint: disable=no-self-use
49        #
50        # Assume modification timestamps for local file systems are
51        # at least precise up to a second.
52        return 1.0
53
54    def fobj(self):
55        """Return a file object for the name/path in the constructor."""
56        return open(self.name, self.mode)
57
58
59class RemoteFile(object):
60    """
61    Represent a file on the remote side which is to be transferred or
62    is already transferred.
63    """
64
65    def __init__(self, ftp_host, name, mode):
66        self._host = ftp_host
67        self._path = ftp_host.path
68        self.name = self._path.abspath(name)
69        self.mode = mode
70
71    def exists(self):
72        """
73        Return `True` if the path representing this file exists.
74        Otherwise return `False`.
75        """
76        return self._path.exists(self.name)
77
78    def mtime(self):
79        """Return the timestamp for the last modification in seconds."""
80        # Convert to client time zone (see definition of time
81        # shift in docstring of `FTPHost.set_time_shift`).
82        return self._path.getmtime(self.name) - self._host.time_shift()
83
84    def mtime_precision(self):
85        """Return the precision of the last modification time in seconds."""
86        # I think using `stat` instead of `lstat` makes more sense here.
87        return self._host.stat(self.name)._st_mtime_precision
88
89    def fobj(self):
90        """Return a file object for the name/path in the constructor."""
91        return self._host.open(self.name, self.mode)
92
93
94def source_is_newer_than_target(source_file, target_file):
95    """
96    Return `True` if the source is newer than the target, else `False`.
97
98    Both arguments are `LocalFile` or `RemoteFile` objects.
99
100    It's assumed that the actual modification time is
101
102      reported_mtime <= actual_mtime <= reported_mtime + mtime_precision
103
104    i. e. that the reported mtime is the actual mtime or rounded down
105    (truncated).
106
107    For the purpose of this test the source is newer than the target
108    if any of the possible actual source modification times is greater
109    than the reported target modification time. In other words: If in
110    doubt, the file should be transferred.
111
112    This is the only situation where the source is _not_ considered
113    newer than the target:
114
115    |/////////////////////|              possible source mtime
116                            |////////|   possible target mtime
117
118    That is, the latest possible actual source modification time is
119    before the first possible actual target modification time.
120    """
121    if source_file.mtime_precision() is ftputil.stat.UNKNOWN_PRECISION:
122        return True
123    else:
124        return (source_file.mtime() + source_file.mtime_precision() >=
125                target_file.mtime())
126
127
128def chunks(fobj, max_chunk_size=MAX_COPY_CHUNK_SIZE):
129    """
130    Return an iterator which yields the contents of the file object.
131
132    For each iteration, at most `max_chunk_size` bytes are read from
133    `fobj` and yielded as a byte string. If the file object is
134    exhausted, then don't yield any more data but stop the iteration,
135    so the client does _not_ get an empty byte string.
136
137    Any exceptions resulting from reading the file object are passed
138    through to the client.
139    """
140    while True:
141        chunk = fobj.read(max_chunk_size)
142        if not chunk:
143            break
144        yield chunk
145
146
147def copyfileobj(source_fobj, target_fobj, max_chunk_size=MAX_COPY_CHUNK_SIZE,
148                callback=None):
149    """Copy data from file-like object source to file-like object target."""
150    # Inspired by `shutil.copyfileobj` (I don't use the `shutil`
151    # code directly because it might change)
152    for chunk in chunks(source_fobj, max_chunk_size):
153        target_fobj.write(chunk)
154        if callback is not None:
155            callback(chunk)
156
157
158def copy_file(source_file, target_file, conditional, callback):
159    """
160    Copy a file from `source_file` to `target_file`.
161
162    These are `LocalFile` or `RemoteFile` objects. Which of them
163    is a local or a remote file, respectively, is determined by
164    the arguments. If `conditional` is true, the file is only
165    copied if the target doesn't exist or is older than the
166    source. If `conditional` is false, the file is copied
167    unconditionally. Return `True` if the file was copied, else
168    `False`.
169    """
170    if conditional:
171        # Evaluate condition: The target file either doesn't exist or is
172        # older than the source file. If in doubt (due to imprecise
173        # timestamps), perform the transfer.
174        transfer_condition = not target_file.exists() or \
175          source_is_newer_than_target(source_file, target_file)
176        if not transfer_condition:
177            # We didn't transfer.
178            return False
179    source_fobj = source_file.fobj()
180    try:
181        target_fobj = target_file.fobj()
182        try:
183            copyfileobj(source_fobj, target_fobj, callback=callback)
184        finally:
185            target_fobj.close()
186    finally:
187        source_fobj.close()
188    # Transfer accomplished
189    return True
Note: See TracBrowser for help on using the repository browser.