1 | # Copyright (C) 2002-2018, Stefan Schwarzer <sschwarzer@sschwarzer.net> |
---|
2 | # and ftputil contributors (see `doc/contributors.txt`) |
---|
3 | # See the file LICENSE for licensing terms. |
---|
4 | |
---|
5 | """ |
---|
6 | `FTPHost` is the central class of the `ftputil` library. |
---|
7 | |
---|
8 | See `__init__.py` for an example. |
---|
9 | """ |
---|
10 | |
---|
11 | import ftplib |
---|
12 | import stat |
---|
13 | import sys |
---|
14 | import time |
---|
15 | |
---|
16 | import ftputil.error |
---|
17 | import ftputil.file |
---|
18 | import ftputil.file_transfer |
---|
19 | import ftputil.path |
---|
20 | import ftputil.stat |
---|
21 | import ftputil.tool |
---|
22 | |
---|
23 | __all__ = ["FTPHost"] |
---|
24 | |
---|
25 | |
---|
26 | # The "protected" attributes PyLint talks about aren't intended for |
---|
27 | # clients of the library. `FTPHost` objects need to use some of these |
---|
28 | # library-internal attributes though. |
---|
29 | # pylint: disable=protected-access |
---|
30 | |
---|
31 | |
---|
32 | ##################################################################### |
---|
33 | # `FTPHost` class with several methods similar to those of `os` |
---|
34 | |
---|
35 | class FTPHost: |
---|
36 | """FTP host class.""" |
---|
37 | |
---|
38 | # Implementation notes: |
---|
39 | # |
---|
40 | # Upon every request of a file (`FTPFile` object) a new FTP |
---|
41 | # session is created (or reused from a cache), leading to a child |
---|
42 | # session of the `FTPHost` object from which the file is requested. |
---|
43 | # |
---|
44 | # This is needed because opening an `FTPFile` will make the |
---|
45 | # local session object wait for the completion of the transfer. |
---|
46 | # In fact, code like this would block indefinitely, if the `RETR` |
---|
47 | # request would be made on the `_session` of the object host: |
---|
48 | # |
---|
49 | # host = FTPHost(ftp_server, user, password) |
---|
50 | # f = host.open("index.html") |
---|
51 | # host.getcwd() # would block! |
---|
52 | # |
---|
53 | # On the other hand, the initially constructed host object will |
---|
54 | # store references to already established `FTPFile` objects and |
---|
55 | # reuse an associated connection if its associated `FTPFile` |
---|
56 | # has been closed. |
---|
57 | |
---|
58 | def __init__(self, *args, **kwargs): |
---|
59 | """Abstract initialization of `FTPHost` object.""" |
---|
60 | # Store arguments for later operations. |
---|
61 | self._args = args |
---|
62 | self._kwargs = kwargs |
---|
63 | #XXX Maybe put the following in a `reset` method. |
---|
64 | # The time shift setting shouldn't be reset though. |
---|
65 | # Make a session according to these arguments. |
---|
66 | self._session = self._make_session() |
---|
67 | # Simulate `os.path`. |
---|
68 | self.path = ftputil.path._Path(self) |
---|
69 | # lstat, stat, listdir services. |
---|
70 | self._stat = ftputil.stat._Stat(self) |
---|
71 | self.stat_cache = self._stat._lstat_cache |
---|
72 | self.stat_cache.enable() |
---|
73 | with ftputil.error.ftplib_error_to_ftp_os_error: |
---|
74 | self._cached_current_dir = \ |
---|
75 | self.path.normpath(ftputil.tool.as_unicode(self._session.pwd())) |
---|
76 | # Associated `FTPHost` objects for data transfer. |
---|
77 | self._children = [] |
---|
78 | # This is only set to something else than `None` if this |
---|
79 | # instance represents an `FTPFile`. |
---|
80 | self._file = None |
---|
81 | # Now opened. |
---|
82 | self.closed = False |
---|
83 | # Set curdir, pardir etc. for the remote host. RFC 959 states |
---|
84 | # that this is, strictly speaking, dependent on the server OS |
---|
85 | # but it seems to work at least with Unix and Windows servers. |
---|
86 | self.curdir, self.pardir, self.sep = ".", "..", "/" |
---|
87 | # Set default time shift (used in `upload_if_newer` and |
---|
88 | # `download_if_newer`). |
---|
89 | self.set_time_shift(0.0) |
---|
90 | # Don't use `LIST -a` option by default. If the server doesn't |
---|
91 | # understand the `-a` option and interprets it as a path, the |
---|
92 | # results can be surprising. See ticket #110. |
---|
93 | self.use_list_a_option = False |
---|
94 | |
---|
95 | def keep_alive(self): |
---|
96 | """ |
---|
97 | Try to keep the connection alive in order to avoid server timeouts. |
---|
98 | |
---|
99 | Note that this won't help if the connection has already timed |
---|
100 | out! In this case, `keep_alive` will raise an `TemporaryError`. |
---|
101 | (Actually, if you get a server timeout, the error - for a specific |
---|
102 | connection - will be permanent.) |
---|
103 | """ |
---|
104 | # Warning: Don't call this method on `FTPHost` instances which |
---|
105 | # represent file transfers. This may fail in confusing ways. |
---|
106 | with ftputil.error.ftplib_error_to_ftp_os_error: |
---|
107 | # Ignore return value. |
---|
108 | self._session.pwd() |
---|
109 | |
---|
110 | # |
---|
111 | # Dealing with child sessions and file-like objects |
---|
112 | # (rather low-level) |
---|
113 | # |
---|
114 | def _make_session(self): |
---|
115 | """ |
---|
116 | Return a new session object according to the current state of |
---|
117 | this `FTPHost` instance. |
---|
118 | """ |
---|
119 | # Don't modify original attributes below. |
---|
120 | args = self._args[:] |
---|
121 | kwargs = self._kwargs.copy() |
---|
122 | # If a session factory has been given on the instantiation of |
---|
123 | # this `FTPHost` object, use the same factory for this |
---|
124 | # `FTPHost` object's child sessions. |
---|
125 | factory = kwargs.pop("session_factory", ftplib.FTP) |
---|
126 | with ftputil.error.ftplib_error_to_ftp_os_error: |
---|
127 | session = factory(*args, **kwargs) |
---|
128 | return session |
---|
129 | |
---|
130 | def _copy(self): |
---|
131 | """Return a copy of this `FTPHost` object.""" |
---|
132 | # The copy includes a new session factory return value (aka |
---|
133 | # session) but doesn't copy the state of `self.getcwd()`. |
---|
134 | return self.__class__(*self._args, **self._kwargs) |
---|
135 | |
---|
136 | def _available_child(self): |
---|
137 | """ |
---|
138 | Return an available (i. e. one whose `_file` object is closed |
---|
139 | and doesn't have a timed-out server connection) child |
---|
140 | (`FTPHost` object) from the pool of children or `None` if |
---|
141 | there aren't any. |
---|
142 | """ |
---|
143 | #TODO: Currently timed-out child sessions aren't removed and |
---|
144 | # may collect over time. In very busy or long running |
---|
145 | # processes, this might slow down an application because the |
---|
146 | # same stale child sessions have to be processed again and |
---|
147 | # again. |
---|
148 | for host in self._children: |
---|
149 | # Test for timeouts only after testing for a closed file: |
---|
150 | # - If a file isn't closed, save time; don't bother to access |
---|
151 | # the remote server. |
---|
152 | # - If a file transfer on the child is in progress, requesting |
---|
153 | # the directory is an invalid operation because of the way |
---|
154 | # the FTP state machine works (see RFC 959). |
---|
155 | if host._file.closed: |
---|
156 | try: |
---|
157 | host._session.pwd() |
---|
158 | # Under high load, a 226 status response from a |
---|
159 | # previous download may arrive too late, so that it's |
---|
160 | # "seen" in the `pwd` call. For now, skip the |
---|
161 | # potential child session; it will be considered again |
---|
162 | # when `_available_child` is called the next time. |
---|
163 | except ftplib.error_reply: |
---|
164 | continue |
---|
165 | # Timed-out sessions raise `error_temp`. |
---|
166 | except ftplib.error_temp: |
---|
167 | continue |
---|
168 | # The server may have closed the connection which may |
---|
169 | # cause `host._session.getline` to raise an `EOFError` |
---|
170 | # (see ticket #114). |
---|
171 | except EOFError: |
---|
172 | continue |
---|
173 | # Under high load, there may be a socket read timeout |
---|
174 | # during the last FTP file `close` (see ticket #112). |
---|
175 | # Note that a socket timeout is quite different from |
---|
176 | # an FTP session timeout. |
---|
177 | except OSError: |
---|
178 | continue |
---|
179 | else: |
---|
180 | # Everything's ok; use this `FTPHost` instance. |
---|
181 | return host |
---|
182 | # Be explicit. |
---|
183 | return None |
---|
184 | |
---|
185 | def open(self, path, mode="r", buffering=None, encoding=None, errors=None, |
---|
186 | newline=None, rest=None): |
---|
187 | """ |
---|
188 | Return an open file(-like) object which is associated with |
---|
189 | this `FTPHost` object. |
---|
190 | |
---|
191 | The arguments `path`, `mode`, `buffering`, `encoding`, |
---|
192 | `errors` and `newlines` have the same meaning as for `open`. |
---|
193 | If `rest` is given as an integer, |
---|
194 | |
---|
195 | - reading will start at the byte (zero-based) `rest` |
---|
196 | - writing will overwrite the remote file from byte `rest` |
---|
197 | |
---|
198 | This method tries to reuse a child but will generate a new one |
---|
199 | if none is available. |
---|
200 | """ |
---|
201 | # Support the same arguments as `open`. |
---|
202 | # pylint: disable=too-many-arguments |
---|
203 | path = ftputil.tool.as_unicode(path) |
---|
204 | host = self._available_child() |
---|
205 | if host is None: |
---|
206 | host = self._copy() |
---|
207 | self._children.append(host) |
---|
208 | host._file = ftputil.file.FTPFile(host) |
---|
209 | basedir = self.getcwd() |
---|
210 | # Prepare for changing the directory (see whitespace workaround |
---|
211 | # in method `_dir`). |
---|
212 | if host.path.isabs(path): |
---|
213 | effective_path = path |
---|
214 | else: |
---|
215 | effective_path = host.path.join(basedir, path) |
---|
216 | effective_dir, effective_file = host.path.split(effective_path) |
---|
217 | try: |
---|
218 | # This will fail if the directory isn't accessible at all. |
---|
219 | host.chdir(effective_dir) |
---|
220 | except ftputil.error.PermanentError: |
---|
221 | # Similarly to a failed `file` in a local file system, |
---|
222 | # raise an `IOError`, not an `OSError`. |
---|
223 | raise ftputil.error.FTPIOError("remote directory '{}' doesn't " |
---|
224 | "exist or has insufficient access rights". |
---|
225 | format(effective_dir)) |
---|
226 | host._file._open(effective_file, mode=mode, buffering=buffering, |
---|
227 | encoding=encoding, errors=errors, newline=newline, |
---|
228 | rest=rest) |
---|
229 | if "w" in mode: |
---|
230 | # Invalidate cache entry because size and timestamps will change. |
---|
231 | self.stat_cache.invalidate(effective_path) |
---|
232 | return host._file |
---|
233 | |
---|
234 | def close(self): |
---|
235 | """Close host connection.""" |
---|
236 | if self.closed: |
---|
237 | return |
---|
238 | # Close associated children. |
---|
239 | for host in self._children: |
---|
240 | # Children have a `_file` attribute which is an `FTPFile` object. |
---|
241 | host._file.close() |
---|
242 | host.close() |
---|
243 | # Now deal with ourself. |
---|
244 | try: |
---|
245 | with ftputil.error.ftplib_error_to_ftp_os_error: |
---|
246 | self._session.close() |
---|
247 | finally: |
---|
248 | # If something went wrong before, the host/session is |
---|
249 | # probably defunct and subsequent calls to `close` won't |
---|
250 | # help either, so consider the host/session closed for |
---|
251 | # practical purposes. |
---|
252 | self.stat_cache.clear() |
---|
253 | self._children = [] |
---|
254 | self.closed = True |
---|
255 | |
---|
256 | # |
---|
257 | # Setting a custom directory parser |
---|
258 | # |
---|
259 | def set_parser(self, parser): |
---|
260 | """ |
---|
261 | Set the parser for extracting stat results from directory |
---|
262 | listings. |
---|
263 | |
---|
264 | The parser interface is described in the documentation, but |
---|
265 | here are the most important things: |
---|
266 | |
---|
267 | - A parser should derive from `ftputil.stat.Parser`. |
---|
268 | |
---|
269 | - The parser has to implement two methods, `parse_line` and |
---|
270 | `ignores_line`. For the latter, there's a probably useful |
---|
271 | default in the class `ftputil.stat.Parser`. |
---|
272 | |
---|
273 | - `parse_line` should try to parse a line of a directory |
---|
274 | listing and return a `ftputil.stat.StatResult` instance. If |
---|
275 | parsing isn't possible, raise `ftputil.error.ParserError` |
---|
276 | with a useful error message. |
---|
277 | |
---|
278 | - `ignores_line` should return a true value if the line isn't |
---|
279 | assumed to contain stat information. |
---|
280 | """ |
---|
281 | # The cache contents, if any, probably aren't useful. |
---|
282 | self.stat_cache.clear() |
---|
283 | # Set the parser explicitly, don't allow "smart" switching anymore. |
---|
284 | self._stat._parser = parser |
---|
285 | self._stat._allow_parser_switching = False |
---|
286 | |
---|
287 | # |
---|
288 | # Time shift adjustment between client (i. e. us) and server |
---|
289 | # |
---|
290 | def set_time_shift(self, time_shift): |
---|
291 | """ |
---|
292 | Set the time shift value (i. e. the time difference between |
---|
293 | client and server) for this `FTPHost` object. By (my) |
---|
294 | definition, the time shift value is positive if the local |
---|
295 | time of the server is greater than the local time of the |
---|
296 | client (for the same physical time), i. e. |
---|
297 | |
---|
298 | time_shift =def= t_server - t_client |
---|
299 | <=> t_server = t_client + time_shift |
---|
300 | <=> t_client = t_server - time_shift |
---|
301 | |
---|
302 | The time shift is measured in seconds. |
---|
303 | """ |
---|
304 | # Implicitly set via `set_time_shift` call in constructor |
---|
305 | # pylint: disable=attribute-defined-outside-init |
---|
306 | self._time_shift = time_shift |
---|
307 | |
---|
308 | def time_shift(self): |
---|
309 | """ |
---|
310 | Return the time shift between FTP server and client. See the |
---|
311 | docstring of `set_time_shift` for more on this value. |
---|
312 | """ |
---|
313 | return self._time_shift |
---|
314 | |
---|
315 | @staticmethod |
---|
316 | def __rounded_time_shift(time_shift): |
---|
317 | """ |
---|
318 | Return the given time shift in seconds, but rounded to |
---|
319 | 15-minute units. The argument is also assumed to be given in |
---|
320 | seconds. |
---|
321 | """ |
---|
322 | minute = 60.0 |
---|
323 | # Avoid division by zero below. |
---|
324 | if time_shift == 0: |
---|
325 | return 0.0 |
---|
326 | # Use a positive value for rounding. |
---|
327 | absolute_time_shift = abs(time_shift) |
---|
328 | signum = time_shift / absolute_time_shift |
---|
329 | # Round absolute time shift to 15-minute units. |
---|
330 | absolute_rounded_time_shift = ( |
---|
331 | int( (absolute_time_shift + (7.5*minute)) / (15.0*minute) ) * |
---|
332 | (15.0*minute)) |
---|
333 | # Return with correct sign. |
---|
334 | return signum * absolute_rounded_time_shift |
---|
335 | |
---|
336 | def __assert_valid_time_shift(self, time_shift): |
---|
337 | """ |
---|
338 | Perform sanity checks on the time shift value (given in |
---|
339 | seconds). If the value is invalid, raise a `TimeShiftError`, |
---|
340 | else simply return `None`. |
---|
341 | """ |
---|
342 | minute = 60.0 # seconds |
---|
343 | hour = 60.0 * minute |
---|
344 | absolute_rounded_time_shift = \ |
---|
345 | abs(self.__rounded_time_shift(time_shift)) |
---|
346 | # Test 1: Fail if the absolute time shift is greater than |
---|
347 | # a full day (24 hours). |
---|
348 | if absolute_rounded_time_shift > 24 * hour: |
---|
349 | raise ftputil.error.TimeShiftError( |
---|
350 | "time shift abs({0:.2f} s) > 1 day".format(time_shift)) |
---|
351 | # Test 2: Fail if the deviation between given time shift and |
---|
352 | # 15-minute units is greater than a certain limit. |
---|
353 | maximum_deviation = 5 * minute |
---|
354 | if abs(time_shift - self.__rounded_time_shift(time_shift)) > \ |
---|
355 | maximum_deviation: |
---|
356 | raise ftputil.error.TimeShiftError( |
---|
357 | "time shift ({0:.2f} s) deviates more than {1:d} s " |
---|
358 | "from 15-minute units".format( |
---|
359 | time_shift, int(maximum_deviation))) |
---|
360 | |
---|
361 | def synchronize_times(self): |
---|
362 | """ |
---|
363 | Synchronize the local times of FTP client and server. This |
---|
364 | is necessary to let `upload_if_newer` and `download_if_newer` |
---|
365 | work correctly. If `synchronize_times` isn't applicable |
---|
366 | (see below), the time shift can still be set explicitly with |
---|
367 | `set_time_shift`. |
---|
368 | |
---|
369 | This implementation of `synchronize_times` requires _all_ of |
---|
370 | the following: |
---|
371 | |
---|
372 | - The connection between server and client is established. |
---|
373 | - The client has write access to the directory that is |
---|
374 | current when `synchronize_times` is called. |
---|
375 | |
---|
376 | The common usage pattern of `synchronize_times` is to call it |
---|
377 | directly after the connection is established. (As can be |
---|
378 | concluded from the points above, this requires write access |
---|
379 | to the login directory.) |
---|
380 | |
---|
381 | If `synchronize_times` fails, it raises a `TimeShiftError`. |
---|
382 | """ |
---|
383 | helper_file_name = "_ftputil_sync_" |
---|
384 | # Open a dummy file for writing in the current directory |
---|
385 | # on the FTP host, then close it. |
---|
386 | try: |
---|
387 | # May raise `FTPIOError` if directory isn't writable. |
---|
388 | file_ = self.open(helper_file_name, "w") |
---|
389 | file_.close() |
---|
390 | except ftputil.error.FTPIOError: |
---|
391 | raise ftputil.error.TimeShiftError( |
---|
392 | """couldn't write helper file in directory '{}'""". |
---|
393 | format(self.getcwd())) |
---|
394 | # If everything worked up to here it should be possible to stat |
---|
395 | # and then remove the just-written file. |
---|
396 | try: |
---|
397 | server_time = self.path.getmtime(helper_file_name) |
---|
398 | self.unlink(helper_file_name) |
---|
399 | except ftputil.error.FTPOSError: |
---|
400 | # If we got a `TimeShiftError` exception above, we should't |
---|
401 | # come here: if we did not get a `TimeShiftError` above, |
---|
402 | # deletion should be possible. The only reason for an exception |
---|
403 | # I can think of here is a race condition by removing write |
---|
404 | # permission from the directory or helper file after it has been |
---|
405 | # written to. |
---|
406 | raise ftputil.error.TimeShiftError( |
---|
407 | "could write helper file but not unlink it") |
---|
408 | # Calculate the difference between server and client. |
---|
409 | now = time.time() |
---|
410 | time_shift = server_time - now |
---|
411 | # As the time shift for this host instance isn't set yet, the |
---|
412 | # directory parser will calculate times one year in the past if |
---|
413 | # the time zone of the server is east from ours. Thus the time |
---|
414 | # shift will be off by a year as well (see ticket #55). |
---|
415 | if time_shift < -360 * 24 * 60 * 60: |
---|
416 | # Re-add one year and re-calculate the time shift. We don't |
---|
417 | # know how many days made up that year (it might have been |
---|
418 | # a leap year), so go the route via `time.localtime` and |
---|
419 | # `time.mktime`. |
---|
420 | server_time_struct = time.localtime(server_time) |
---|
421 | server_time_struct = (server_time_struct.tm_year+1,) + \ |
---|
422 | server_time_struct[1:] |
---|
423 | server_time = time.mktime(server_time_struct) |
---|
424 | time_shift = server_time - now |
---|
425 | # Do some sanity checks. |
---|
426 | self.__assert_valid_time_shift(time_shift) |
---|
427 | # If tests passed, store the time difference as time shift value. |
---|
428 | self.set_time_shift(self.__rounded_time_shift(time_shift)) |
---|
429 | |
---|
430 | # |
---|
431 | # Operations based on file-like objects (rather high-level), |
---|
432 | # like upload and download |
---|
433 | # |
---|
434 | # This code doesn't complain if the chunk size is passed as a |
---|
435 | # positional argument but emits a deprecation warning if `length` |
---|
436 | # is used as a keyword argument. |
---|
437 | @staticmethod |
---|
438 | def copyfileobj(source, target, |
---|
439 | max_chunk_size=ftputil.file_transfer.MAX_COPY_CHUNK_SIZE, |
---|
440 | callback=None): |
---|
441 | """ |
---|
442 | Copy data from file-like object `source` to file-like object |
---|
443 | `target`. |
---|
444 | """ |
---|
445 | ftputil.file_transfer.copyfileobj(source, target, max_chunk_size, |
---|
446 | callback) |
---|
447 | |
---|
448 | def _upload_files(self, source_path, target_path): |
---|
449 | """ |
---|
450 | Return a `LocalFile` and `RemoteFile` as source and target, |
---|
451 | respectively. |
---|
452 | |
---|
453 | The strings `source_path` and `target_path` are the (absolute |
---|
454 | or relative) paths of the local and the remote file, respectively. |
---|
455 | """ |
---|
456 | source_file = ftputil.file_transfer.LocalFile(source_path, "rb") |
---|
457 | # Passing `self` (the `FTPHost` instance) here is correct. |
---|
458 | target_file = ftputil.file_transfer.RemoteFile(self, target_path, "wb") |
---|
459 | return source_file, target_file |
---|
460 | |
---|
461 | def upload(self, source, target, callback=None): |
---|
462 | """ |
---|
463 | Upload a file from the local source (name) to the remote |
---|
464 | target (name). |
---|
465 | |
---|
466 | If a callable `callback` is given, it's called after every |
---|
467 | chunk of transferred data. The chunk size is a constant |
---|
468 | defined in `file_transfer`. The callback will be called with a |
---|
469 | single argument, the data chunk that was transferred before |
---|
470 | the callback was called. |
---|
471 | """ |
---|
472 | target = ftputil.tool.as_unicode(target) |
---|
473 | source_file, target_file = self._upload_files(source, target) |
---|
474 | ftputil.file_transfer.copy_file(source_file, target_file, |
---|
475 | conditional=False, callback=callback) |
---|
476 | |
---|
477 | def upload_if_newer(self, source, target, callback=None): |
---|
478 | """ |
---|
479 | Upload a file only if it's newer than the target on the |
---|
480 | remote host or if the target file does not exist. See the |
---|
481 | method `upload` for the meaning of the parameters. |
---|
482 | |
---|
483 | If an upload was necessary, return `True`, else return `False`. |
---|
484 | |
---|
485 | If a callable `callback` is given, it's called after every |
---|
486 | chunk of transferred data. The chunk size is a constant |
---|
487 | defined in `file_transfer`. The callback will be called with a |
---|
488 | single argument, the data chunk that was transferred before |
---|
489 | the callback was called. |
---|
490 | """ |
---|
491 | target = ftputil.tool.as_unicode(target) |
---|
492 | source_file, target_file = self._upload_files(source, target) |
---|
493 | return ftputil.file_transfer.copy_file(source_file, target_file, |
---|
494 | conditional=True, |
---|
495 | callback=callback) |
---|
496 | |
---|
497 | def _download_files(self, source_path, target_path): |
---|
498 | """ |
---|
499 | Return a `RemoteFile` and `LocalFile` as source and target, |
---|
500 | respectively. |
---|
501 | |
---|
502 | The strings `source_path` and `target_path` are the (absolute |
---|
503 | or relative) paths of the remote and the local file, respectively. |
---|
504 | """ |
---|
505 | source_file = ftputil.file_transfer.RemoteFile(self, source_path, "rb") |
---|
506 | target_file = ftputil.file_transfer.LocalFile(target_path, "wb") |
---|
507 | return source_file, target_file |
---|
508 | |
---|
509 | def download(self, source, target, callback=None): |
---|
510 | """ |
---|
511 | Download a file from the remote source (name) to the local |
---|
512 | target (name). |
---|
513 | |
---|
514 | If a callable `callback` is given, it's called after every |
---|
515 | chunk of transferred data. The chunk size is a constant |
---|
516 | defined in `file_transfer`. The callback will be called with a |
---|
517 | single argument, the data chunk that was transferred before |
---|
518 | the callback was called. |
---|
519 | """ |
---|
520 | source = ftputil.tool.as_unicode(source) |
---|
521 | source_file, target_file = self._download_files(source, target) |
---|
522 | ftputil.file_transfer.copy_file(source_file, target_file, |
---|
523 | conditional=False, callback=callback) |
---|
524 | |
---|
525 | def download_if_newer(self, source, target, callback=None): |
---|
526 | """ |
---|
527 | Download a file only if it's newer than the target on the |
---|
528 | local host or if the target file does not exist. See the |
---|
529 | method `download` for the meaning of the parameters. |
---|
530 | |
---|
531 | If a download was necessary, return `True`, else return |
---|
532 | `False`. |
---|
533 | |
---|
534 | If a callable `callback` is given, it's called after every |
---|
535 | chunk of transferred data. The chunk size is a constant |
---|
536 | defined in `file_transfer`. The callback will be called with a |
---|
537 | single argument, the data chunk that was transferred before |
---|
538 | the callback was called. |
---|
539 | """ |
---|
540 | source = ftputil.tool.as_unicode(source) |
---|
541 | source_file, target_file = self._download_files(source, target) |
---|
542 | return ftputil.file_transfer.copy_file(source_file, target_file, |
---|
543 | conditional=True, |
---|
544 | callback=callback) |
---|
545 | |
---|
546 | # |
---|
547 | # Helper methods to descend into a directory before executing a command |
---|
548 | # |
---|
549 | def _check_inaccessible_login_directory(self): |
---|
550 | """ |
---|
551 | Raise an `InaccessibleLoginDirError` exception if we can't |
---|
552 | change to the login directory. This test is only reliable if |
---|
553 | the current directory is the login directory. |
---|
554 | """ |
---|
555 | presumable_login_dir = self.getcwd() |
---|
556 | # Bail out with an internal error rather than modify the |
---|
557 | # current directory without hope of restoration. |
---|
558 | try: |
---|
559 | self.chdir(presumable_login_dir) |
---|
560 | except ftputil.error.PermanentError: |
---|
561 | raise ftputil.error.InaccessibleLoginDirError( |
---|
562 | "directory '{}' is not accessible". |
---|
563 | format(presumable_login_dir)) |
---|
564 | |
---|
565 | def _robust_ftp_command(self, command, path, descend_deeply=False): |
---|
566 | """ |
---|
567 | Run an FTP command on a path. The return value of the method |
---|
568 | is the return value of the command. |
---|
569 | |
---|
570 | If `descend_deeply` is true (the default is false), descend |
---|
571 | deeply, i. e. change the directory to the end of the path. |
---|
572 | """ |
---|
573 | # If we can't change to the yet-current directory, the code |
---|
574 | # below won't work (see below), so in this case rather raise |
---|
575 | # an exception than giving wrong results. |
---|
576 | self._check_inaccessible_login_directory() |
---|
577 | # Some FTP servers don't behave as expected if the directory |
---|
578 | # portion of the path contains whitespace; some even yield |
---|
579 | # strange results if the command isn't executed in the |
---|
580 | # current directory. Therefore, change to the directory |
---|
581 | # which contains the item to run the command on and invoke |
---|
582 | # the command just there. |
---|
583 | # |
---|
584 | # Remember old working directory. |
---|
585 | old_dir = self.getcwd() |
---|
586 | try: |
---|
587 | if descend_deeply: |
---|
588 | # Invoke the command in (not: on) the deepest directory. |
---|
589 | self.chdir(path) |
---|
590 | # Workaround for some servers that give recursive |
---|
591 | # listings when called with a dot as path; see issue #33, |
---|
592 | # http://ftputil.sschwarzer.net/trac/ticket/33 |
---|
593 | return command(self, "") |
---|
594 | else: |
---|
595 | # Invoke the command in the "next to last" directory. |
---|
596 | head, tail = self.path.split(path) |
---|
597 | self.chdir(head) |
---|
598 | return command(self, tail) |
---|
599 | finally: |
---|
600 | self.chdir(old_dir) |
---|
601 | |
---|
602 | # |
---|
603 | # Miscellaneous utility methods resembling functions in `os` |
---|
604 | # |
---|
605 | def getcwd(self): |
---|
606 | """Return the current path name.""" |
---|
607 | return self._cached_current_dir |
---|
608 | |
---|
609 | def chdir(self, path): |
---|
610 | """Change the directory on the host.""" |
---|
611 | path = ftputil.tool.as_unicode(path) |
---|
612 | with ftputil.error.ftplib_error_to_ftp_os_error: |
---|
613 | self._session.cwd(path) |
---|
614 | # The path given as the argument is relative to the old current |
---|
615 | # directory, therefore join them. |
---|
616 | self._cached_current_dir = \ |
---|
617 | self.path.normpath(self.path.join(self._cached_current_dir, path)) |
---|
618 | |
---|
619 | # Ignore unused argument `mode` |
---|
620 | # pylint: disable=unused-argument |
---|
621 | def mkdir(self, path, mode=None): |
---|
622 | """ |
---|
623 | Make the directory path on the remote host. The argument |
---|
624 | `mode` is ignored and only "supported" for similarity with |
---|
625 | `os.mkdir`. |
---|
626 | """ |
---|
627 | path = ftputil.tool.as_unicode(path) |
---|
628 | def command(self, path): |
---|
629 | """Callback function.""" |
---|
630 | with ftputil.error.ftplib_error_to_ftp_os_error: |
---|
631 | self._session.mkd(path) |
---|
632 | self._robust_ftp_command(command, path) |
---|
633 | |
---|
634 | # TODO: The virtual directory support doesn't have unit tests yet |
---|
635 | # because the mocking most likely would be quite complicated. The |
---|
636 | # tests should be added when mainly the `mock` library is used |
---|
637 | # instead of the mock code in `test.mock_ftplib`. |
---|
638 | # |
---|
639 | # Ignore unused argument `mode` |
---|
640 | # pylint: disable=unused-argument |
---|
641 | def makedirs(self, path, mode=None): |
---|
642 | """ |
---|
643 | Make the directory `path`, but also make not yet existing |
---|
644 | intermediate directories, like `os.makedirs`. The value of |
---|
645 | `mode` is only accepted for compatibility with `os.makedirs` |
---|
646 | but otherwise ignored. |
---|
647 | """ |
---|
648 | path = ftputil.tool.as_unicode(path) |
---|
649 | path = self.path.abspath(path) |
---|
650 | directories = path.split(self.sep) |
---|
651 | old_dir = self.getcwd() |
---|
652 | try: |
---|
653 | # Try to build the directory chain from the "uppermost" to |
---|
654 | # the "lowermost" directory. |
---|
655 | for index in range(1, len(directories)): |
---|
656 | # Re-insert the separator which got lost by using |
---|
657 | # `path.split`. |
---|
658 | next_directory = (self.sep + |
---|
659 | self.path.join(*directories[:index+1])) |
---|
660 | # If we have "virtual directories" (see #86), just |
---|
661 | # listing the parent directory won't tell us if a |
---|
662 | # directory actually exists. So try to change into the |
---|
663 | # directory. |
---|
664 | try: |
---|
665 | self.chdir(next_directory) |
---|
666 | except ftputil.error.PermanentError: |
---|
667 | try: |
---|
668 | self.mkdir(next_directory) |
---|
669 | except ftputil.error.PermanentError: |
---|
670 | # Find out the cause of the error. Re-raise |
---|
671 | # the exception only if the directory didn't |
---|
672 | # exist already, else something went _really_ |
---|
673 | # wrong, e. g. there's a regular file with the |
---|
674 | # name of the directory. |
---|
675 | if not self.path.isdir(next_directory): |
---|
676 | raise |
---|
677 | finally: |
---|
678 | self.chdir(old_dir) |
---|
679 | |
---|
680 | def rmdir(self, path): |
---|
681 | """ |
---|
682 | Remove the _empty_ directory `path` on the remote host. |
---|
683 | |
---|
684 | Compatibility note: |
---|
685 | |
---|
686 | Previous versions of ftputil could possibly delete non- |
---|
687 | empty directories as well, - if the server allowed it. This |
---|
688 | is no longer supported. |
---|
689 | """ |
---|
690 | path = ftputil.tool.as_unicode(path) |
---|
691 | path = self.path.abspath(path) |
---|
692 | if self.listdir(path): |
---|
693 | raise ftputil.error.PermanentError("directory '{}' not empty". |
---|
694 | format(path)) |
---|
695 | #XXX How does `rmd` work with links? |
---|
696 | def command(self, path): |
---|
697 | """Callback function.""" |
---|
698 | with ftputil.error.ftplib_error_to_ftp_os_error: |
---|
699 | self._session.rmd(path) |
---|
700 | self._robust_ftp_command(command, path) |
---|
701 | self.stat_cache.invalidate(path) |
---|
702 | |
---|
703 | def remove(self, path): |
---|
704 | """ |
---|
705 | Remove the file or link given by `path`. |
---|
706 | |
---|
707 | Raise a `PermanentError` if the path doesn't exist, but maybe |
---|
708 | raise other exceptions depending on the state of the server |
---|
709 | (e. g. timeout). |
---|
710 | """ |
---|
711 | path = ftputil.tool.as_unicode(path) |
---|
712 | path = self.path.abspath(path) |
---|
713 | # Though `isfile` includes also links to files, `islink` |
---|
714 | # is needed to include links to directories. |
---|
715 | if self.path.isfile(path) or self.path.islink(path) or \ |
---|
716 | not self.path.exists(path): |
---|
717 | # If the path doesn't exist, let the removal command trigger |
---|
718 | # an exception with a more appropriate error message. |
---|
719 | def command(self, path): |
---|
720 | """Callback function.""" |
---|
721 | with ftputil.error.ftplib_error_to_ftp_os_error: |
---|
722 | self._session.delete(path) |
---|
723 | self._robust_ftp_command(command, path) |
---|
724 | else: |
---|
725 | raise ftputil.error.PermanentError( |
---|
726 | "remove/unlink can only delete files and links, " |
---|
727 | "not directories") |
---|
728 | self.stat_cache.invalidate(path) |
---|
729 | |
---|
730 | unlink = remove |
---|
731 | |
---|
732 | def rmtree(self, path, ignore_errors=False, onerror=None): |
---|
733 | """ |
---|
734 | Remove the given remote, possibly non-empty, directory tree. |
---|
735 | The interface of this method is rather complex, in favor of |
---|
736 | compatibility with `shutil.rmtree`. |
---|
737 | |
---|
738 | If `ignore_errors` is set to a true value, errors are ignored. |
---|
739 | If `ignore_errors` is a false value _and_ `onerror` isn't set, |
---|
740 | all exceptions occurring during the tree iteration and |
---|
741 | processing are raised. These exceptions are all of type |
---|
742 | `PermanentError`. |
---|
743 | |
---|
744 | To distinguish between error situations, pass in a callable |
---|
745 | for `onerror`. This callable must accept three arguments: |
---|
746 | `func`, `path` and `exc_info`. `func` is a bound method |
---|
747 | object, _for example_ `your_host_object.listdir`. `path` is |
---|
748 | the path that was the recent argument of the respective method |
---|
749 | (`listdir`, `remove`, `rmdir`). `exc_info` is the exception |
---|
750 | info as it's got from `sys.exc_info`. |
---|
751 | |
---|
752 | Implementation note: The code is copied from `shutil.rmtree` |
---|
753 | in Python 2.4 and adapted to ftputil. |
---|
754 | """ |
---|
755 | path = ftputil.tool.as_unicode(path) |
---|
756 | # The following code is an adapted version of Python 2.4's |
---|
757 | # `shutil.rmtree` function. |
---|
758 | if ignore_errors: |
---|
759 | def new_onerror(*args): |
---|
760 | """Do nothing.""" |
---|
761 | # pylint: disable=unused-argument |
---|
762 | pass |
---|
763 | elif onerror is None: |
---|
764 | def new_onerror(*args): |
---|
765 | """Re-raise exception.""" |
---|
766 | # pylint: disable=unused-argument |
---|
767 | raise |
---|
768 | else: |
---|
769 | new_onerror = onerror |
---|
770 | names = [] |
---|
771 | try: |
---|
772 | names = self.listdir(path) |
---|
773 | except ftputil.error.PermanentError: |
---|
774 | new_onerror(self.listdir, path, sys.exc_info()) |
---|
775 | for name in names: |
---|
776 | full_name = self.path.join(path, name) |
---|
777 | try: |
---|
778 | mode = self.lstat(full_name).st_mode |
---|
779 | except ftputil.error.PermanentError: |
---|
780 | mode = 0 |
---|
781 | if stat.S_ISDIR(mode): |
---|
782 | self.rmtree(full_name, ignore_errors, new_onerror) |
---|
783 | else: |
---|
784 | try: |
---|
785 | self.remove(full_name) |
---|
786 | except ftputil.error.PermanentError: |
---|
787 | new_onerror(self.remove, full_name, sys.exc_info()) |
---|
788 | try: |
---|
789 | self.rmdir(path) |
---|
790 | except ftputil.error.FTPOSError: |
---|
791 | new_onerror(self.rmdir, path, sys.exc_info()) |
---|
792 | |
---|
793 | def rename(self, source, target): |
---|
794 | """Rename the source on the FTP host to target.""" |
---|
795 | source = ftputil.tool.as_unicode(source) |
---|
796 | target = ftputil.tool.as_unicode(target) |
---|
797 | # The following code is in spirit similar to the code in the |
---|
798 | # method `_robust_ftp_command`, though we do _not_ do |
---|
799 | # _everything_ imaginable. |
---|
800 | self._check_inaccessible_login_directory() |
---|
801 | source_head, source_tail = self.path.split(source) |
---|
802 | target_head, target_tail = self.path.split(target) |
---|
803 | paths_contain_whitespace = (" " in source_head) or (" " in target_head) |
---|
804 | if paths_contain_whitespace and source_head == target_head: |
---|
805 | # Both items are in the same directory. |
---|
806 | old_dir = self.getcwd() |
---|
807 | try: |
---|
808 | self.chdir(source_head) |
---|
809 | with ftputil.error.ftplib_error_to_ftp_os_error: |
---|
810 | self._session.rename(source_tail, target_tail) |
---|
811 | finally: |
---|
812 | self.chdir(old_dir) |
---|
813 | else: |
---|
814 | # Use straightforward command. |
---|
815 | with ftputil.error.ftplib_error_to_ftp_os_error: |
---|
816 | self._session.rename(source, target) |
---|
817 | |
---|
818 | #XXX One could argue to put this method into the `_Stat` class, but |
---|
819 | # I refrained from that because then `_Stat` would have to know |
---|
820 | # about `FTPHost`'s `_session` attribute and in turn about |
---|
821 | # `_session`'s `dir` method. |
---|
822 | def _dir(self, path): |
---|
823 | """Return a directory listing as made by FTP's `LIST` command.""" |
---|
824 | # Don't use `self.path.isdir` in this method because that |
---|
825 | # would cause a call of `(l)stat` and thus a call to `_dir`, |
---|
826 | # so we would end up with an infinite recursion. |
---|
827 | def _FTPHost_dir_command(self, path): |
---|
828 | """Callback function.""" |
---|
829 | lines = [] |
---|
830 | def callback(line): |
---|
831 | """Callback function.""" |
---|
832 | lines.append(ftputil.tool.as_unicode(line)) |
---|
833 | with ftputil.error.ftplib_error_to_ftp_os_error: |
---|
834 | if self.use_list_a_option: |
---|
835 | self._session.dir("-a", path, callback) |
---|
836 | else: |
---|
837 | self._session.dir(path, callback) |
---|
838 | return lines |
---|
839 | lines = self._robust_ftp_command(_FTPHost_dir_command, path, |
---|
840 | descend_deeply=True) |
---|
841 | return lines |
---|
842 | |
---|
843 | # The `listdir`, `lstat` and `stat` methods don't use |
---|
844 | # `_robust_ftp_command` because they implicitly already use |
---|
845 | # `_dir` which actually uses `_robust_ftp_command`. |
---|
846 | def listdir(self, path): |
---|
847 | """ |
---|
848 | Return a list of directories, files etc. in the directory |
---|
849 | named `path`. |
---|
850 | |
---|
851 | If the directory listing from the server can't be parsed with |
---|
852 | any of the available parsers raise a `ParserError`. |
---|
853 | """ |
---|
854 | original_path = path |
---|
855 | path = ftputil.tool.as_unicode(path) |
---|
856 | items = self._stat._listdir(path) |
---|
857 | return [ftputil.tool.same_string_type_as(original_path, item) |
---|
858 | for item in items] |
---|
859 | |
---|
860 | def lstat(self, path, _exception_for_missing_path=True): |
---|
861 | """ |
---|
862 | Return an object similar to that returned by `os.lstat`. |
---|
863 | |
---|
864 | If the directory listing from the server can't be parsed with |
---|
865 | any of the available parsers, raise a `ParserError`. If the |
---|
866 | directory _can_ be parsed and the `path` is _not_ found, raise |
---|
867 | a `PermanentError`. |
---|
868 | |
---|
869 | (`_exception_for_missing_path` is an implementation aid and |
---|
870 | _not_ intended for use by ftputil clients.) |
---|
871 | """ |
---|
872 | path = ftputil.tool.as_unicode(path) |
---|
873 | return self._stat._lstat(path, _exception_for_missing_path) |
---|
874 | |
---|
875 | def stat(self, path, _exception_for_missing_path=True): |
---|
876 | """ |
---|
877 | Return info from a "stat" call on `path`. |
---|
878 | |
---|
879 | If the directory containing `path` can't be parsed, raise a |
---|
880 | `ParserError`. If the directory containing `path` can be |
---|
881 | parsed but the `path` can't be found, raise a |
---|
882 | `PermanentError`. Also raise a `PermanentError` if there's an |
---|
883 | endless (cyclic) chain of symbolic links "behind" the `path`. |
---|
884 | |
---|
885 | (`_exception_for_missing_path` is an implementation aid and |
---|
886 | _not_ intended for use by ftputil clients.) |
---|
887 | """ |
---|
888 | path = ftputil.tool.as_unicode(path) |
---|
889 | return self._stat._stat(path, _exception_for_missing_path) |
---|
890 | |
---|
891 | def walk(self, top, topdown=True, onerror=None, followlinks=False): |
---|
892 | """ |
---|
893 | Iterate over directory tree and return a tuple (dirpath, |
---|
894 | dirnames, filenames) on each iteration, like the `os.walk` |
---|
895 | function (see https://docs.python.org/library/os.html#os.walk ). |
---|
896 | """ |
---|
897 | top = ftputil.tool.as_unicode(top) |
---|
898 | # The following code is copied from `os.walk` in Python 2.4 |
---|
899 | # and adapted to ftputil. |
---|
900 | try: |
---|
901 | names = self.listdir(top) |
---|
902 | except ftputil.error.FTPOSError as err: |
---|
903 | if onerror is not None: |
---|
904 | onerror(err) |
---|
905 | return |
---|
906 | dirs, nondirs = [], [] |
---|
907 | for name in names: |
---|
908 | if self.path.isdir(self.path.join(top, name)): |
---|
909 | dirs.append(name) |
---|
910 | else: |
---|
911 | nondirs.append(name) |
---|
912 | if topdown: |
---|
913 | yield top, dirs, nondirs |
---|
914 | for name in dirs: |
---|
915 | path = self.path.join(top, name) |
---|
916 | if followlinks or not self.path.islink(path): |
---|
917 | for item in self.walk(path, topdown, onerror, followlinks): |
---|
918 | yield item |
---|
919 | if not topdown: |
---|
920 | yield top, dirs, nondirs |
---|
921 | |
---|
922 | def chmod(self, path, mode): |
---|
923 | """ |
---|
924 | Change the mode of a remote `path` (a string) to the integer |
---|
925 | `mode`. This integer uses the same bits as the mode value |
---|
926 | returned by the `stat` and `lstat` commands. |
---|
927 | |
---|
928 | If something goes wrong, raise a `TemporaryError` or a |
---|
929 | `PermanentError`, according to the status code returned by |
---|
930 | the server. In particular, a non-existent path usually |
---|
931 | causes a `PermanentError`. |
---|
932 | """ |
---|
933 | path = ftputil.tool.as_unicode(path) |
---|
934 | path = self.path.abspath(path) |
---|
935 | def command(self, path): |
---|
936 | """Callback function.""" |
---|
937 | with ftputil.error.ftplib_error_to_ftp_os_error: |
---|
938 | self._session.voidcmd("SITE CHMOD 0{0:o} {1}". |
---|
939 | format(mode, path)) |
---|
940 | self._robust_ftp_command(command, path) |
---|
941 | self.stat_cache.invalidate(path) |
---|
942 | |
---|
943 | def __getstate__(self): |
---|
944 | raise TypeError("cannot serialize FTPHost object") |
---|
945 | |
---|
946 | # |
---|
947 | # Context manager methods |
---|
948 | # |
---|
949 | def __enter__(self): |
---|
950 | # Return `self`, so it can be accessed as the variable |
---|
951 | # component of the `with` statement. |
---|
952 | return self |
---|
953 | |
---|
954 | def __exit__(self, exc_type, exc_val, exc_tb): |
---|
955 | # We don't need the `exc_*` arguments here. |
---|
956 | # pylint: disable=unused-argument |
---|
957 | self.close() |
---|
958 | # Be explicit. |
---|
959 | return False |
---|