1 | # Copyright (C) 2002-2018, Stefan Schwarzer <sschwarzer@sschwarzer.net> |
---|
2 | # and ftputil contributors (see `doc/contributors.txt`) |
---|
3 | # See the file LICENSE for licensing terms. |
---|
4 | |
---|
5 | """ |
---|
6 | `FTPHost` is the central class of the `ftputil` library. |
---|
7 | |
---|
8 | See `__init__.py` for an example. |
---|
9 | """ |
---|
10 | |
---|
11 | import ftplib |
---|
12 | import stat |
---|
13 | import sys |
---|
14 | import time |
---|
15 | import warnings |
---|
16 | |
---|
17 | import ftputil.error |
---|
18 | import ftputil.file |
---|
19 | import ftputil.file_transfer |
---|
20 | import ftputil.path |
---|
21 | import ftputil.stat |
---|
22 | import ftputil.tool |
---|
23 | |
---|
24 | __all__ = ["FTPHost"] |
---|
25 | |
---|
26 | |
---|
27 | # The "protected" attributes PyLint talks about aren't intended for |
---|
28 | # clients of the library. `FTPHost` objects need to use some of these |
---|
29 | # library-internal attributes though. |
---|
30 | # pylint: disable=protected-access |
---|
31 | |
---|
32 | |
---|
33 | ##################################################################### |
---|
34 | # `FTPHost` class with several methods similar to those of `os` |
---|
35 | |
---|
36 | class FTPHost: |
---|
37 | """FTP host class.""" |
---|
38 | |
---|
39 | # Implementation notes: |
---|
40 | # |
---|
41 | # Upon every request of a file (`FTPFile` object) a new FTP |
---|
42 | # session is created (or reused from a cache), leading to a child |
---|
43 | # session of the `FTPHost` object from which the file is requested. |
---|
44 | # |
---|
45 | # This is needed because opening an `FTPFile` will make the |
---|
46 | # local session object wait for the completion of the transfer. |
---|
47 | # In fact, code like this would block indefinitely, if the `RETR` |
---|
48 | # request would be made on the `_session` of the object host: |
---|
49 | # |
---|
50 | # host = FTPHost(ftp_server, user, password) |
---|
51 | # f = host.open("index.html") |
---|
52 | # host.getcwd() # would block! |
---|
53 | # |
---|
54 | # On the other hand, the initially constructed host object will |
---|
55 | # store references to already established `FTPFile` objects and |
---|
56 | # reuse an associated connection if its associated `FTPFile` |
---|
57 | # has been closed. |
---|
58 | |
---|
59 | def __init__(self, *args, **kwargs): |
---|
60 | """Abstract initialization of `FTPHost` object.""" |
---|
61 | # Store arguments for later operations. |
---|
62 | self._args = args |
---|
63 | self._kwargs = kwargs |
---|
64 | #XXX Maybe put the following in a `reset` method. |
---|
65 | # The time shift setting shouldn't be reset though. |
---|
66 | # Make a session according to these arguments. |
---|
67 | self._session = self._make_session() |
---|
68 | # Simulate `os.path`. |
---|
69 | self.path = ftputil.path._Path(self) |
---|
70 | # lstat, stat, listdir services. |
---|
71 | self._stat = ftputil.stat._Stat(self) |
---|
72 | self.stat_cache = self._stat._lstat_cache |
---|
73 | self.stat_cache.enable() |
---|
74 | with ftputil.error.ftplib_error_to_ftp_os_error: |
---|
75 | self._cached_current_dir = \ |
---|
76 | self.path.normpath(ftputil.tool.as_unicode(self._session.pwd())) |
---|
77 | # Associated `FTPHost` objects for data transfer. |
---|
78 | self._children = [] |
---|
79 | # This is only set to something else than `None` if this |
---|
80 | # instance represents an `FTPFile`. |
---|
81 | self._file = None |
---|
82 | # Now opened. |
---|
83 | self.closed = False |
---|
84 | # Set curdir, pardir etc. for the remote host. RFC 959 states |
---|
85 | # that this is, strictly speaking, dependent on the server OS |
---|
86 | # but it seems to work at least with Unix and Windows servers. |
---|
87 | self.curdir, self.pardir, self.sep = ".", "..", "/" |
---|
88 | # Set default time shift (used in `upload_if_newer` and |
---|
89 | # `download_if_newer`). |
---|
90 | self.set_time_shift(0.0) |
---|
91 | # Don't use `LIST -a` option by default. If the server doesn't |
---|
92 | # understand the `-a` option and interprets it as a path, the |
---|
93 | # results can be surprising. See ticket #110. |
---|
94 | self.use_list_a_option = False |
---|
95 | |
---|
96 | def keep_alive(self): |
---|
97 | """ |
---|
98 | Try to keep the connection alive in order to avoid server timeouts. |
---|
99 | |
---|
100 | Note that this won't help if the connection has already timed |
---|
101 | out! In this case, `keep_alive` will raise an `TemporaryError`. |
---|
102 | (Actually, if you get a server timeout, the error - for a specific |
---|
103 | connection - will be permanent.) |
---|
104 | """ |
---|
105 | # Warning: Don't call this method on `FTPHost` instances which |
---|
106 | # represent file transfers. This may fail in confusing ways. |
---|
107 | with ftputil.error.ftplib_error_to_ftp_os_error: |
---|
108 | # Ignore return value. |
---|
109 | self._session.pwd() |
---|
110 | |
---|
111 | # |
---|
112 | # Dealing with child sessions and file-like objects |
---|
113 | # (rather low-level) |
---|
114 | # |
---|
115 | def _make_session(self): |
---|
116 | """ |
---|
117 | Return a new session object according to the current state of |
---|
118 | this `FTPHost` instance. |
---|
119 | """ |
---|
120 | # Don't modify original attributes below. |
---|
121 | args = self._args[:] |
---|
122 | kwargs = self._kwargs.copy() |
---|
123 | # If a session factory has been given on the instantiation of |
---|
124 | # this `FTPHost` object, use the same factory for this |
---|
125 | # `FTPHost` object's child sessions. |
---|
126 | factory = kwargs.pop("session_factory", ftplib.FTP) |
---|
127 | with ftputil.error.ftplib_error_to_ftp_os_error: |
---|
128 | session = factory(*args, **kwargs) |
---|
129 | return session |
---|
130 | |
---|
131 | def _copy(self): |
---|
132 | """Return a copy of this `FTPHost` object.""" |
---|
133 | # The copy includes a new session factory return value (aka |
---|
134 | # session) but doesn't copy the state of `self.getcwd()`. |
---|
135 | return self.__class__(*self._args, **self._kwargs) |
---|
136 | |
---|
137 | def _available_child(self): |
---|
138 | """ |
---|
139 | Return an available (i. e. one whose `_file` object is closed |
---|
140 | and doesn't have a timed-out server connection) child |
---|
141 | (`FTPHost` object) from the pool of children or `None` if |
---|
142 | there aren't any. |
---|
143 | """ |
---|
144 | #TODO: Currently timed-out child sessions aren't removed and |
---|
145 | # may collect over time. In very busy or long running |
---|
146 | # processes, this might slow down an application because the |
---|
147 | # same stale child sessions have to be processed again and |
---|
148 | # again. |
---|
149 | for host in self._children: |
---|
150 | # Test for timeouts only after testing for a closed file: |
---|
151 | # - If a file isn't closed, save time; don't bother to access |
---|
152 | # the remote server. |
---|
153 | # - If a file transfer on the child is in progress, requesting |
---|
154 | # the directory is an invalid operation because of the way |
---|
155 | # the FTP state machine works (see RFC 959). |
---|
156 | if host._file.closed: |
---|
157 | try: |
---|
158 | host._session.pwd() |
---|
159 | # Under high load, a 226 status response from a |
---|
160 | # previous download may arrive too late, so that it's |
---|
161 | # "seen" in the `pwd` call. For now, skip the |
---|
162 | # potential child session; it will be considered again |
---|
163 | # when `_available_child` is called the next time. |
---|
164 | except ftplib.error_reply: |
---|
165 | continue |
---|
166 | # Timed-out sessions raise `error_temp`. |
---|
167 | except ftplib.error_temp: |
---|
168 | continue |
---|
169 | # The server may have closed the connection which may |
---|
170 | # cause `host._session.getline` to raise an `EOFError` |
---|
171 | # (see ticket #114). |
---|
172 | except EOFError: |
---|
173 | continue |
---|
174 | # Under high load, there may be a socket read timeout |
---|
175 | # during the last FTP file `close` (see ticket #112). |
---|
176 | # Note that a socket timeout is quite different from |
---|
177 | # an FTP session timeout. |
---|
178 | except OSError: |
---|
179 | continue |
---|
180 | else: |
---|
181 | # Everything's ok; use this `FTPHost` instance. |
---|
182 | return host |
---|
183 | # Be explicit. |
---|
184 | return None |
---|
185 | |
---|
186 | def open(self, path, mode="r", buffering=None, encoding=None, errors=None, |
---|
187 | newline=None, rest=None): |
---|
188 | """ |
---|
189 | Return an open file(-like) object which is associated with |
---|
190 | this `FTPHost` object. |
---|
191 | |
---|
192 | The arguments `path`, `mode`, `buffering`, `encoding`, |
---|
193 | `errors` and `newlines` have the same meaning as for `open`. |
---|
194 | If `rest` is given as an integer, |
---|
195 | |
---|
196 | - reading will start at the byte (zero-based) `rest` |
---|
197 | - writing will overwrite the remote file from byte `rest` |
---|
198 | |
---|
199 | This method tries to reuse a child but will generate a new one |
---|
200 | if none is available. |
---|
201 | """ |
---|
202 | # Support the same arguments as `open`. |
---|
203 | # pylint: disable=too-many-arguments |
---|
204 | path = ftputil.tool.as_unicode(path) |
---|
205 | host = self._available_child() |
---|
206 | if host is None: |
---|
207 | host = self._copy() |
---|
208 | self._children.append(host) |
---|
209 | host._file = ftputil.file.FTPFile(host) |
---|
210 | basedir = self.getcwd() |
---|
211 | # Prepare for changing the directory (see whitespace workaround |
---|
212 | # in method `_dir`). |
---|
213 | if host.path.isabs(path): |
---|
214 | effective_path = path |
---|
215 | else: |
---|
216 | effective_path = host.path.join(basedir, path) |
---|
217 | effective_dir, effective_file = host.path.split(effective_path) |
---|
218 | try: |
---|
219 | # This will fail if the directory isn't accessible at all. |
---|
220 | host.chdir(effective_dir) |
---|
221 | except ftputil.error.PermanentError: |
---|
222 | # Similarly to a failed `file` in a local file system, |
---|
223 | # raise an `IOError`, not an `OSError`. |
---|
224 | raise ftputil.error.FTPIOError("remote directory '{0}' doesn't " |
---|
225 | "exist or has insufficient access rights". |
---|
226 | format(effective_dir)) |
---|
227 | host._file._open(effective_file, mode=mode, buffering=buffering, |
---|
228 | encoding=encoding, errors=errors, newline=newline, |
---|
229 | rest=rest) |
---|
230 | if "w" in mode: |
---|
231 | # Invalidate cache entry because size and timestamps will change. |
---|
232 | self.stat_cache.invalidate(effective_path) |
---|
233 | return host._file |
---|
234 | |
---|
235 | def close(self): |
---|
236 | """Close host connection.""" |
---|
237 | if self.closed: |
---|
238 | return |
---|
239 | # Close associated children. |
---|
240 | for host in self._children: |
---|
241 | # Children have a `_file` attribute which is an `FTPFile` object. |
---|
242 | host._file.close() |
---|
243 | host.close() |
---|
244 | # Now deal with ourself. |
---|
245 | try: |
---|
246 | with ftputil.error.ftplib_error_to_ftp_os_error: |
---|
247 | self._session.close() |
---|
248 | finally: |
---|
249 | # If something went wrong before, the host/session is |
---|
250 | # probably defunct and subsequent calls to `close` won't |
---|
251 | # help either, so consider the host/session closed for |
---|
252 | # practical purposes. |
---|
253 | self.stat_cache.clear() |
---|
254 | self._children = [] |
---|
255 | self.closed = True |
---|
256 | |
---|
257 | # |
---|
258 | # Setting a custom directory parser |
---|
259 | # |
---|
260 | def set_parser(self, parser): |
---|
261 | """ |
---|
262 | Set the parser for extracting stat results from directory |
---|
263 | listings. |
---|
264 | |
---|
265 | The parser interface is described in the documentation, but |
---|
266 | here are the most important things: |
---|
267 | |
---|
268 | - A parser should derive from `ftputil.stat.Parser`. |
---|
269 | |
---|
270 | - The parser has to implement two methods, `parse_line` and |
---|
271 | `ignores_line`. For the latter, there's a probably useful |
---|
272 | default in the class `ftputil.stat.Parser`. |
---|
273 | |
---|
274 | - `parse_line` should try to parse a line of a directory |
---|
275 | listing and return a `ftputil.stat.StatResult` instance. If |
---|
276 | parsing isn't possible, raise `ftputil.error.ParserError` |
---|
277 | with a useful error message. |
---|
278 | |
---|
279 | - `ignores_line` should return a true value if the line isn't |
---|
280 | assumed to contain stat information. |
---|
281 | """ |
---|
282 | # The cache contents, if any, probably aren't useful. |
---|
283 | self.stat_cache.clear() |
---|
284 | # Set the parser explicitly, don't allow "smart" switching anymore. |
---|
285 | self._stat._parser = parser |
---|
286 | self._stat._allow_parser_switching = False |
---|
287 | |
---|
288 | # |
---|
289 | # Time shift adjustment between client (i. e. us) and server |
---|
290 | # |
---|
291 | def set_time_shift(self, time_shift): |
---|
292 | """ |
---|
293 | Set the time shift value (i. e. the time difference between |
---|
294 | client and server) for this `FTPHost` object. By (my) |
---|
295 | definition, the time shift value is positive if the local |
---|
296 | time of the server is greater than the local time of the |
---|
297 | client (for the same physical time), i. e. |
---|
298 | |
---|
299 | time_shift =def= t_server - t_client |
---|
300 | <=> t_server = t_client + time_shift |
---|
301 | <=> t_client = t_server - time_shift |
---|
302 | |
---|
303 | The time shift is measured in seconds. |
---|
304 | """ |
---|
305 | # Implicitly set via `set_time_shift` call in constructor |
---|
306 | # pylint: disable=attribute-defined-outside-init |
---|
307 | self._time_shift = time_shift |
---|
308 | |
---|
309 | def time_shift(self): |
---|
310 | """ |
---|
311 | Return the time shift between FTP server and client. See the |
---|
312 | docstring of `set_time_shift` for more on this value. |
---|
313 | """ |
---|
314 | return self._time_shift |
---|
315 | |
---|
316 | @staticmethod |
---|
317 | def __rounded_time_shift(time_shift): |
---|
318 | """ |
---|
319 | Return the given time shift in seconds, but rounded to |
---|
320 | full hours. The argument is also assumed to be given in |
---|
321 | seconds. |
---|
322 | """ |
---|
323 | minute = 60.0 |
---|
324 | hour = 60.0 * minute |
---|
325 | # Avoid division by zero below. |
---|
326 | if time_shift == 0: |
---|
327 | return 0.0 |
---|
328 | # Use a positive value for rounding. |
---|
329 | absolute_time_shift = abs(time_shift) |
---|
330 | signum = time_shift / absolute_time_shift |
---|
331 | # Round absolute time shift to 15-minute units. |
---|
332 | absolute_rounded_time_shift = ( |
---|
333 | int( (absolute_time_shift + (7.5*minute)) / (15.0*minute) ) * |
---|
334 | (15.0*minute)) |
---|
335 | # Return with correct sign. |
---|
336 | return signum * absolute_rounded_time_shift |
---|
337 | |
---|
338 | def __assert_valid_time_shift(self, time_shift): |
---|
339 | """ |
---|
340 | Perform sanity checks on the time shift value (given in |
---|
341 | seconds). If the value is invalid, raise a `TimeShiftError`, |
---|
342 | else simply return `None`. |
---|
343 | """ |
---|
344 | minute = 60.0 # seconds |
---|
345 | hour = 60.0 * minute |
---|
346 | absolute_rounded_time_shift = \ |
---|
347 | abs(self.__rounded_time_shift(time_shift)) |
---|
348 | # Test 1: Fail if the absolute time shift is greater than |
---|
349 | # a full day (24 hours). |
---|
350 | if absolute_rounded_time_shift > 24 * hour: |
---|
351 | raise ftputil.error.TimeShiftError( |
---|
352 | "time shift abs({0:.2f} s) > 1 day".format(time_shift)) |
---|
353 | # Test 2: Fail if the deviation between given time shift and |
---|
354 | # 15-minute units is greater than a certain limit. |
---|
355 | maximum_deviation = 5 * minute |
---|
356 | if abs(time_shift - self.__rounded_time_shift(time_shift)) > \ |
---|
357 | maximum_deviation: |
---|
358 | raise ftputil.error.TimeShiftError( |
---|
359 | "time shift ({0:.2f} s) deviates more than {1:d} s " |
---|
360 | "from 15-minute units".format( |
---|
361 | time_shift, int(maximum_deviation))) |
---|
362 | |
---|
363 | def synchronize_times(self): |
---|
364 | """ |
---|
365 | Synchronize the local times of FTP client and server. This |
---|
366 | is necessary to let `upload_if_newer` and `download_if_newer` |
---|
367 | work correctly. If `synchronize_times` isn't applicable |
---|
368 | (see below), the time shift can still be set explicitly with |
---|
369 | `set_time_shift`. |
---|
370 | |
---|
371 | This implementation of `synchronize_times` requires _all_ of |
---|
372 | the following: |
---|
373 | |
---|
374 | - The connection between server and client is established. |
---|
375 | - The client has write access to the directory that is |
---|
376 | current when `synchronize_times` is called. |
---|
377 | |
---|
378 | The common usage pattern of `synchronize_times` is to call it |
---|
379 | directly after the connection is established. (As can be |
---|
380 | concluded from the points above, this requires write access |
---|
381 | to the login directory.) |
---|
382 | |
---|
383 | If `synchronize_times` fails, it raises a `TimeShiftError`. |
---|
384 | """ |
---|
385 | helper_file_name = "_ftputil_sync_" |
---|
386 | # Open a dummy file for writing in the current directory |
---|
387 | # on the FTP host, then close it. |
---|
388 | try: |
---|
389 | # May raise `FTPIOError` if directory isn't writable. |
---|
390 | file_ = self.open(helper_file_name, "w") |
---|
391 | file_.close() |
---|
392 | except ftputil.error.FTPIOError: |
---|
393 | raise ftputil.error.TimeShiftError( |
---|
394 | """couldn't write helper file in directory '{0}'""". |
---|
395 | format(self.getcwd())) |
---|
396 | # If everything worked up to here it should be possible to stat |
---|
397 | # and then remove the just-written file. |
---|
398 | try: |
---|
399 | server_time = self.path.getmtime(helper_file_name) |
---|
400 | self.unlink(helper_file_name) |
---|
401 | except ftputil.error.FTPOSError: |
---|
402 | # If we got a `TimeShiftError` exception above, we should't |
---|
403 | # come here: if we did not get a `TimeShiftError` above, |
---|
404 | # deletion should be possible. The only reason for an exception |
---|
405 | # I can think of here is a race condition by removing write |
---|
406 | # permission from the directory or helper file after it has been |
---|
407 | # written to. |
---|
408 | raise ftputil.error.TimeShiftError( |
---|
409 | "could write helper file but not unlink it") |
---|
410 | # Calculate the difference between server and client. |
---|
411 | now = time.time() |
---|
412 | time_shift = server_time - now |
---|
413 | # As the time shift for this host instance isn't set yet, the |
---|
414 | # directory parser will calculate times one year in the past if |
---|
415 | # the time zone of the server is east from ours. Thus the time |
---|
416 | # shift will be off by a year as well (see ticket #55). |
---|
417 | if time_shift < -360 * 24 * 60 * 60: |
---|
418 | # Re-add one year and re-calculate the time shift. We don't |
---|
419 | # know how many days made up that year (it might have been |
---|
420 | # a leap year), so go the route via `time.localtime` and |
---|
421 | # `time.mktime`. |
---|
422 | server_time_struct = time.localtime(server_time) |
---|
423 | server_time_struct = (server_time_struct.tm_year+1,) + \ |
---|
424 | server_time_struct[1:] |
---|
425 | server_time = time.mktime(server_time_struct) |
---|
426 | time_shift = server_time - now |
---|
427 | # Do some sanity checks. |
---|
428 | self.__assert_valid_time_shift(time_shift) |
---|
429 | # If tests passed, store the time difference as time shift value. |
---|
430 | self.set_time_shift(self.__rounded_time_shift(time_shift)) |
---|
431 | |
---|
432 | # |
---|
433 | # Operations based on file-like objects (rather high-level), |
---|
434 | # like upload and download |
---|
435 | # |
---|
436 | # This code doesn't complain if the chunk size is passed as a |
---|
437 | # positional argument but emits a deprecation warning if `length` |
---|
438 | # is used as a keyword argument. |
---|
439 | @staticmethod |
---|
440 | def copyfileobj(source, target, |
---|
441 | max_chunk_size=ftputil.file_transfer.MAX_COPY_CHUNK_SIZE, |
---|
442 | callback=None): |
---|
443 | """ |
---|
444 | Copy data from file-like object `source` to file-like object |
---|
445 | `target`. |
---|
446 | """ |
---|
447 | ftputil.file_transfer.copyfileobj(source, target, max_chunk_size, |
---|
448 | callback) |
---|
449 | |
---|
450 | def _upload_files(self, source_path, target_path): |
---|
451 | """ |
---|
452 | Return a `LocalFile` and `RemoteFile` as source and target, |
---|
453 | respectively. |
---|
454 | |
---|
455 | The strings `source_path` and `target_path` are the (absolute |
---|
456 | or relative) paths of the local and the remote file, respectively. |
---|
457 | """ |
---|
458 | source_file = ftputil.file_transfer.LocalFile(source_path, "rb") |
---|
459 | # Passing `self` (the `FTPHost` instance) here is correct. |
---|
460 | target_file = ftputil.file_transfer.RemoteFile(self, target_path, "wb") |
---|
461 | return source_file, target_file |
---|
462 | |
---|
463 | def upload(self, source, target, callback=None): |
---|
464 | """ |
---|
465 | Upload a file from the local source (name) to the remote |
---|
466 | target (name). |
---|
467 | |
---|
468 | If a callable `callback` is given, it's called after every |
---|
469 | chunk of transferred data. The chunk size is a constant |
---|
470 | defined in `file_transfer`. The callback will be called with a |
---|
471 | single argument, the data chunk that was transferred before |
---|
472 | the callback was called. |
---|
473 | """ |
---|
474 | target = ftputil.tool.as_unicode(target) |
---|
475 | source_file, target_file = self._upload_files(source, target) |
---|
476 | ftputil.file_transfer.copy_file(source_file, target_file, |
---|
477 | conditional=False, callback=callback) |
---|
478 | |
---|
479 | def upload_if_newer(self, source, target, callback=None): |
---|
480 | """ |
---|
481 | Upload a file only if it's newer than the target on the |
---|
482 | remote host or if the target file does not exist. See the |
---|
483 | method `upload` for the meaning of the parameters. |
---|
484 | |
---|
485 | If an upload was necessary, return `True`, else return `False`. |
---|
486 | |
---|
487 | If a callable `callback` is given, it's called after every |
---|
488 | chunk of transferred data. The chunk size is a constant |
---|
489 | defined in `file_transfer`. The callback will be called with a |
---|
490 | single argument, the data chunk that was transferred before |
---|
491 | the callback was called. |
---|
492 | """ |
---|
493 | target = ftputil.tool.as_unicode(target) |
---|
494 | source_file, target_file = self._upload_files(source, target) |
---|
495 | return ftputil.file_transfer.copy_file(source_file, target_file, |
---|
496 | conditional=True, |
---|
497 | callback=callback) |
---|
498 | |
---|
499 | def _download_files(self, source_path, target_path): |
---|
500 | """ |
---|
501 | Return a `RemoteFile` and `LocalFile` as source and target, |
---|
502 | respectively. |
---|
503 | |
---|
504 | The strings `source_path` and `target_path` are the (absolute |
---|
505 | or relative) paths of the remote and the local file, respectively. |
---|
506 | """ |
---|
507 | source_file = ftputil.file_transfer.RemoteFile(self, source_path, "rb") |
---|
508 | target_file = ftputil.file_transfer.LocalFile(target_path, "wb") |
---|
509 | return source_file, target_file |
---|
510 | |
---|
511 | def download(self, source, target, callback=None): |
---|
512 | """ |
---|
513 | Download a file from the remote source (name) to the local |
---|
514 | target (name). |
---|
515 | |
---|
516 | If a callable `callback` is given, it's called after every |
---|
517 | chunk of transferred data. The chunk size is a constant |
---|
518 | defined in `file_transfer`. The callback will be called with a |
---|
519 | single argument, the data chunk that was transferred before |
---|
520 | the callback was called. |
---|
521 | """ |
---|
522 | source = ftputil.tool.as_unicode(source) |
---|
523 | source_file, target_file = self._download_files(source, target) |
---|
524 | ftputil.file_transfer.copy_file(source_file, target_file, |
---|
525 | conditional=False, callback=callback) |
---|
526 | |
---|
527 | def download_if_newer(self, source, target, callback=None): |
---|
528 | """ |
---|
529 | Download a file only if it's newer than the target on the |
---|
530 | local host or if the target file does not exist. See the |
---|
531 | method `download` for the meaning of the parameters. |
---|
532 | |
---|
533 | If a download was necessary, return `True`, else return |
---|
534 | `False`. |
---|
535 | |
---|
536 | If a callable `callback` is given, it's called after every |
---|
537 | chunk of transferred data. The chunk size is a constant |
---|
538 | defined in `file_transfer`. The callback will be called with a |
---|
539 | single argument, the data chunk that was transferred before |
---|
540 | the callback was called. |
---|
541 | """ |
---|
542 | source = ftputil.tool.as_unicode(source) |
---|
543 | source_file, target_file = self._download_files(source, target) |
---|
544 | return ftputil.file_transfer.copy_file(source_file, target_file, |
---|
545 | conditional=True, |
---|
546 | callback=callback) |
---|
547 | |
---|
548 | # |
---|
549 | # Helper methods to descend into a directory before executing a command |
---|
550 | # |
---|
551 | def _check_inaccessible_login_directory(self): |
---|
552 | """ |
---|
553 | Raise an `InaccessibleLoginDirError` exception if we can't |
---|
554 | change to the login directory. This test is only reliable if |
---|
555 | the current directory is the login directory. |
---|
556 | """ |
---|
557 | presumable_login_dir = self.getcwd() |
---|
558 | # Bail out with an internal error rather than modify the |
---|
559 | # current directory without hope of restoration. |
---|
560 | try: |
---|
561 | self.chdir(presumable_login_dir) |
---|
562 | except ftputil.error.PermanentError: |
---|
563 | raise ftputil.error.InaccessibleLoginDirError( |
---|
564 | "directory '{0}' is not accessible". |
---|
565 | format(presumable_login_dir)) |
---|
566 | |
---|
567 | def _robust_ftp_command(self, command, path, descend_deeply=False): |
---|
568 | """ |
---|
569 | Run an FTP command on a path. The return value of the method |
---|
570 | is the return value of the command. |
---|
571 | |
---|
572 | If `descend_deeply` is true (the default is false), descend |
---|
573 | deeply, i. e. change the directory to the end of the path. |
---|
574 | """ |
---|
575 | # If we can't change to the yet-current directory, the code |
---|
576 | # below won't work (see below), so in this case rather raise |
---|
577 | # an exception than giving wrong results. |
---|
578 | self._check_inaccessible_login_directory() |
---|
579 | # Some FTP servers don't behave as expected if the directory |
---|
580 | # portion of the path contains whitespace; some even yield |
---|
581 | # strange results if the command isn't executed in the |
---|
582 | # current directory. Therefore, change to the directory |
---|
583 | # which contains the item to run the command on and invoke |
---|
584 | # the command just there. |
---|
585 | # |
---|
586 | # Remember old working directory. |
---|
587 | old_dir = self.getcwd() |
---|
588 | try: |
---|
589 | if descend_deeply: |
---|
590 | # Invoke the command in (not: on) the deepest directory. |
---|
591 | self.chdir(path) |
---|
592 | # Workaround for some servers that give recursive |
---|
593 | # listings when called with a dot as path; see issue #33, |
---|
594 | # http://ftputil.sschwarzer.net/trac/ticket/33 |
---|
595 | return command(self, "") |
---|
596 | else: |
---|
597 | # Invoke the command in the "next to last" directory. |
---|
598 | head, tail = self.path.split(path) |
---|
599 | self.chdir(head) |
---|
600 | return command(self, tail) |
---|
601 | finally: |
---|
602 | self.chdir(old_dir) |
---|
603 | |
---|
604 | # |
---|
605 | # Miscellaneous utility methods resembling functions in `os` |
---|
606 | # |
---|
607 | def getcwd(self): |
---|
608 | """Return the current path name.""" |
---|
609 | return self._cached_current_dir |
---|
610 | |
---|
611 | def chdir(self, path): |
---|
612 | """Change the directory on the host.""" |
---|
613 | path = ftputil.tool.as_unicode(path) |
---|
614 | with ftputil.error.ftplib_error_to_ftp_os_error: |
---|
615 | self._session.cwd(path) |
---|
616 | # The path given as the argument is relative to the old current |
---|
617 | # directory, therefore join them. |
---|
618 | self._cached_current_dir = \ |
---|
619 | self.path.normpath(self.path.join(self._cached_current_dir, path)) |
---|
620 | |
---|
621 | # Ignore unused argument `mode` |
---|
622 | # pylint: disable=unused-argument |
---|
623 | def mkdir(self, path, mode=None): |
---|
624 | """ |
---|
625 | Make the directory path on the remote host. The argument |
---|
626 | `mode` is ignored and only "supported" for similarity with |
---|
627 | `os.mkdir`. |
---|
628 | """ |
---|
629 | path = ftputil.tool.as_unicode(path) |
---|
630 | def command(self, path): |
---|
631 | """Callback function.""" |
---|
632 | with ftputil.error.ftplib_error_to_ftp_os_error: |
---|
633 | self._session.mkd(path) |
---|
634 | self._robust_ftp_command(command, path) |
---|
635 | |
---|
636 | # TODO: The virtual directory support doesn't have unit tests yet |
---|
637 | # because the mocking most likely would be quite complicated. The |
---|
638 | # tests should be added when mainly the `mock` library is used |
---|
639 | # instead of the mock code in `test.mock_ftplib`. |
---|
640 | # |
---|
641 | # Ignore unused argument `mode` |
---|
642 | # pylint: disable=unused-argument |
---|
643 | def makedirs(self, path, mode=None): |
---|
644 | """ |
---|
645 | Make the directory `path`, but also make not yet existing |
---|
646 | intermediate directories, like `os.makedirs`. The value of |
---|
647 | `mode` is only accepted for compatibility with `os.makedirs` |
---|
648 | but otherwise ignored. |
---|
649 | """ |
---|
650 | path = ftputil.tool.as_unicode(path) |
---|
651 | path = self.path.abspath(path) |
---|
652 | directories = path.split(self.sep) |
---|
653 | old_dir = self.getcwd() |
---|
654 | try: |
---|
655 | # Try to build the directory chain from the "uppermost" to |
---|
656 | # the "lowermost" directory. |
---|
657 | for index in range(1, len(directories)): |
---|
658 | # Re-insert the separator which got lost by using |
---|
659 | # `path.split`. |
---|
660 | next_directory = (self.sep + |
---|
661 | self.path.join(*directories[:index+1])) |
---|
662 | # If we have "virtual directories" (see #86), just |
---|
663 | # listing the parent directory won't tell us if a |
---|
664 | # directory actually exists. So try to change into the |
---|
665 | # directory. |
---|
666 | try: |
---|
667 | self.chdir(next_directory) |
---|
668 | except ftputil.error.PermanentError: |
---|
669 | try: |
---|
670 | self.mkdir(next_directory) |
---|
671 | except ftputil.error.PermanentError: |
---|
672 | # Find out the cause of the error. Re-raise |
---|
673 | # the exception only if the directory didn't |
---|
674 | # exist already, else something went _really_ |
---|
675 | # wrong, e. g. there's a regular file with the |
---|
676 | # name of the directory. |
---|
677 | if not self.path.isdir(next_directory): |
---|
678 | raise |
---|
679 | finally: |
---|
680 | self.chdir(old_dir) |
---|
681 | |
---|
682 | def rmdir(self, path): |
---|
683 | """ |
---|
684 | Remove the _empty_ directory `path` on the remote host. |
---|
685 | |
---|
686 | Compatibility note: |
---|
687 | |
---|
688 | Previous versions of ftputil could possibly delete non- |
---|
689 | empty directories as well, - if the server allowed it. This |
---|
690 | is no longer supported. |
---|
691 | """ |
---|
692 | path = ftputil.tool.as_unicode(path) |
---|
693 | path = self.path.abspath(path) |
---|
694 | if self.listdir(path): |
---|
695 | raise ftputil.error.PermanentError("directory '{0}' not empty". |
---|
696 | format(path)) |
---|
697 | #XXX How does `rmd` work with links? |
---|
698 | def command(self, path): |
---|
699 | """Callback function.""" |
---|
700 | with ftputil.error.ftplib_error_to_ftp_os_error: |
---|
701 | self._session.rmd(path) |
---|
702 | self._robust_ftp_command(command, path) |
---|
703 | self.stat_cache.invalidate(path) |
---|
704 | |
---|
705 | def remove(self, path): |
---|
706 | """ |
---|
707 | Remove the file or link given by `path`. |
---|
708 | |
---|
709 | Raise a `PermanentError` if the path doesn't exist, but maybe |
---|
710 | raise other exceptions depending on the state of the server |
---|
711 | (e. g. timeout). |
---|
712 | """ |
---|
713 | path = ftputil.tool.as_unicode(path) |
---|
714 | path = self.path.abspath(path) |
---|
715 | # Though `isfile` includes also links to files, `islink` |
---|
716 | # is needed to include links to directories. |
---|
717 | if self.path.isfile(path) or self.path.islink(path) or \ |
---|
718 | not self.path.exists(path): |
---|
719 | # If the path doesn't exist, let the removal command trigger |
---|
720 | # an exception with a more appropriate error message. |
---|
721 | def command(self, path): |
---|
722 | """Callback function.""" |
---|
723 | with ftputil.error.ftplib_error_to_ftp_os_error: |
---|
724 | self._session.delete(path) |
---|
725 | self._robust_ftp_command(command, path) |
---|
726 | else: |
---|
727 | raise ftputil.error.PermanentError( |
---|
728 | "remove/unlink can only delete files and links, " |
---|
729 | "not directories") |
---|
730 | self.stat_cache.invalidate(path) |
---|
731 | |
---|
732 | unlink = remove |
---|
733 | |
---|
734 | def rmtree(self, path, ignore_errors=False, onerror=None): |
---|
735 | """ |
---|
736 | Remove the given remote, possibly non-empty, directory tree. |
---|
737 | The interface of this method is rather complex, in favor of |
---|
738 | compatibility with `shutil.rmtree`. |
---|
739 | |
---|
740 | If `ignore_errors` is set to a true value, errors are ignored. |
---|
741 | If `ignore_errors` is a false value _and_ `onerror` isn't set, |
---|
742 | all exceptions occurring during the tree iteration and |
---|
743 | processing are raised. These exceptions are all of type |
---|
744 | `PermanentError`. |
---|
745 | |
---|
746 | To distinguish between error situations, pass in a callable |
---|
747 | for `onerror`. This callable must accept three arguments: |
---|
748 | `func`, `path` and `exc_info`. `func` is a bound method |
---|
749 | object, _for example_ `your_host_object.listdir`. `path` is |
---|
750 | the path that was the recent argument of the respective method |
---|
751 | (`listdir`, `remove`, `rmdir`). `exc_info` is the exception |
---|
752 | info as it's got from `sys.exc_info`. |
---|
753 | |
---|
754 | Implementation note: The code is copied from `shutil.rmtree` |
---|
755 | in Python 2.4 and adapted to ftputil. |
---|
756 | """ |
---|
757 | path = ftputil.tool.as_unicode(path) |
---|
758 | # The following code is an adapted version of Python 2.4's |
---|
759 | # `shutil.rmtree` function. |
---|
760 | if ignore_errors: |
---|
761 | def new_onerror(*args): |
---|
762 | """Do nothing.""" |
---|
763 | # pylint: disable=unused-argument |
---|
764 | pass |
---|
765 | elif onerror is None: |
---|
766 | def new_onerror(*args): |
---|
767 | """Re-raise exception.""" |
---|
768 | # pylint: disable=unused-argument |
---|
769 | raise |
---|
770 | else: |
---|
771 | new_onerror = onerror |
---|
772 | names = [] |
---|
773 | try: |
---|
774 | names = self.listdir(path) |
---|
775 | except ftputil.error.PermanentError: |
---|
776 | new_onerror(self.listdir, path, sys.exc_info()) |
---|
777 | for name in names: |
---|
778 | full_name = self.path.join(path, name) |
---|
779 | try: |
---|
780 | mode = self.lstat(full_name).st_mode |
---|
781 | except ftputil.error.PermanentError: |
---|
782 | mode = 0 |
---|
783 | if stat.S_ISDIR(mode): |
---|
784 | self.rmtree(full_name, ignore_errors, new_onerror) |
---|
785 | else: |
---|
786 | try: |
---|
787 | self.remove(full_name) |
---|
788 | except ftputil.error.PermanentError: |
---|
789 | new_onerror(self.remove, full_name, sys.exc_info()) |
---|
790 | try: |
---|
791 | self.rmdir(path) |
---|
792 | except ftputil.error.FTPOSError: |
---|
793 | new_onerror(self.rmdir, path, sys.exc_info()) |
---|
794 | |
---|
795 | def rename(self, source, target): |
---|
796 | """Rename the source on the FTP host to target.""" |
---|
797 | source = ftputil.tool.as_unicode(source) |
---|
798 | target = ftputil.tool.as_unicode(target) |
---|
799 | # The following code is in spirit similar to the code in the |
---|
800 | # method `_robust_ftp_command`, though we do _not_ do |
---|
801 | # _everything_ imaginable. |
---|
802 | self._check_inaccessible_login_directory() |
---|
803 | source_head, source_tail = self.path.split(source) |
---|
804 | target_head, target_tail = self.path.split(target) |
---|
805 | paths_contain_whitespace = (" " in source_head) or (" " in target_head) |
---|
806 | if paths_contain_whitespace and source_head == target_head: |
---|
807 | # Both items are in the same directory. |
---|
808 | old_dir = self.getcwd() |
---|
809 | try: |
---|
810 | self.chdir(source_head) |
---|
811 | with ftputil.error.ftplib_error_to_ftp_os_error: |
---|
812 | self._session.rename(source_tail, target_tail) |
---|
813 | finally: |
---|
814 | self.chdir(old_dir) |
---|
815 | else: |
---|
816 | # Use straightforward command. |
---|
817 | with ftputil.error.ftplib_error_to_ftp_os_error: |
---|
818 | self._session.rename(source, target) |
---|
819 | |
---|
820 | #XXX One could argue to put this method into the `_Stat` class, but |
---|
821 | # I refrained from that because then `_Stat` would have to know |
---|
822 | # about `FTPHost`'s `_session` attribute and in turn about |
---|
823 | # `_session`'s `dir` method. |
---|
824 | def _dir(self, path): |
---|
825 | """Return a directory listing as made by FTP's `LIST` command.""" |
---|
826 | # Don't use `self.path.isdir` in this method because that |
---|
827 | # would cause a call of `(l)stat` and thus a call to `_dir`, |
---|
828 | # so we would end up with an infinite recursion. |
---|
829 | def _FTPHost_dir_command(self, path): |
---|
830 | """Callback function.""" |
---|
831 | lines = [] |
---|
832 | def callback(line): |
---|
833 | """Callback function.""" |
---|
834 | lines.append(ftputil.tool.as_unicode(line)) |
---|
835 | with ftputil.error.ftplib_error_to_ftp_os_error: |
---|
836 | if self.use_list_a_option: |
---|
837 | self._session.dir("-a", path, callback) |
---|
838 | else: |
---|
839 | self._session.dir(path, callback) |
---|
840 | return lines |
---|
841 | lines = self._robust_ftp_command(_FTPHost_dir_command, path, |
---|
842 | descend_deeply=True) |
---|
843 | return lines |
---|
844 | |
---|
845 | # The `listdir`, `lstat` and `stat` methods don't use |
---|
846 | # `_robust_ftp_command` because they implicitly already use |
---|
847 | # `_dir` which actually uses `_robust_ftp_command`. |
---|
848 | def listdir(self, path): |
---|
849 | """ |
---|
850 | Return a list of directories, files etc. in the directory |
---|
851 | named `path`. |
---|
852 | |
---|
853 | If the directory listing from the server can't be parsed with |
---|
854 | any of the available parsers raise a `ParserError`. |
---|
855 | """ |
---|
856 | original_path = path |
---|
857 | path = ftputil.tool.as_unicode(path) |
---|
858 | items = self._stat._listdir(path) |
---|
859 | return [ftputil.tool.same_string_type_as(original_path, item) |
---|
860 | for item in items] |
---|
861 | |
---|
862 | def lstat(self, path, _exception_for_missing_path=True): |
---|
863 | """ |
---|
864 | Return an object similar to that returned by `os.lstat`. |
---|
865 | |
---|
866 | If the directory listing from the server can't be parsed with |
---|
867 | any of the available parsers, raise a `ParserError`. If the |
---|
868 | directory _can_ be parsed and the `path` is _not_ found, raise |
---|
869 | a `PermanentError`. |
---|
870 | |
---|
871 | (`_exception_for_missing_path` is an implementation aid and |
---|
872 | _not_ intended for use by ftputil clients.) |
---|
873 | """ |
---|
874 | path = ftputil.tool.as_unicode(path) |
---|
875 | return self._stat._lstat(path, _exception_for_missing_path) |
---|
876 | |
---|
877 | def stat(self, path, _exception_for_missing_path=True): |
---|
878 | """ |
---|
879 | Return info from a "stat" call on `path`. |
---|
880 | |
---|
881 | If the directory containing `path` can't be parsed, raise a |
---|
882 | `ParserError`. If the directory containing `path` can be |
---|
883 | parsed but the `path` can't be found, raise a |
---|
884 | `PermanentError`. Also raise a `PermanentError` if there's an |
---|
885 | endless (cyclic) chain of symbolic links "behind" the `path`. |
---|
886 | |
---|
887 | (`_exception_for_missing_path` is an implementation aid and |
---|
888 | _not_ intended for use by ftputil clients.) |
---|
889 | """ |
---|
890 | path = ftputil.tool.as_unicode(path) |
---|
891 | return self._stat._stat(path, _exception_for_missing_path) |
---|
892 | |
---|
893 | def walk(self, top, topdown=True, onerror=None, followlinks=False): |
---|
894 | """ |
---|
895 | Iterate over directory tree and return a tuple (dirpath, |
---|
896 | dirnames, filenames) on each iteration, like the `os.walk` |
---|
897 | function (see https://docs.python.org/library/os.html#os.walk ). |
---|
898 | """ |
---|
899 | top = ftputil.tool.as_unicode(top) |
---|
900 | # The following code is copied from `os.walk` in Python 2.4 |
---|
901 | # and adapted to ftputil. |
---|
902 | try: |
---|
903 | names = self.listdir(top) |
---|
904 | except ftputil.error.FTPOSError as err: |
---|
905 | if onerror is not None: |
---|
906 | onerror(err) |
---|
907 | return |
---|
908 | dirs, nondirs = [], [] |
---|
909 | for name in names: |
---|
910 | if self.path.isdir(self.path.join(top, name)): |
---|
911 | dirs.append(name) |
---|
912 | else: |
---|
913 | nondirs.append(name) |
---|
914 | if topdown: |
---|
915 | yield top, dirs, nondirs |
---|
916 | for name in dirs: |
---|
917 | path = self.path.join(top, name) |
---|
918 | if followlinks or not self.path.islink(path): |
---|
919 | for item in self.walk(path, topdown, onerror, followlinks): |
---|
920 | yield item |
---|
921 | if not topdown: |
---|
922 | yield top, dirs, nondirs |
---|
923 | |
---|
924 | def chmod(self, path, mode): |
---|
925 | """ |
---|
926 | Change the mode of a remote `path` (a string) to the integer |
---|
927 | `mode`. This integer uses the same bits as the mode value |
---|
928 | returned by the `stat` and `lstat` commands. |
---|
929 | |
---|
930 | If something goes wrong, raise a `TemporaryError` or a |
---|
931 | `PermanentError`, according to the status code returned by |
---|
932 | the server. In particular, a non-existent path usually |
---|
933 | causes a `PermanentError`. |
---|
934 | """ |
---|
935 | path = ftputil.tool.as_unicode(path) |
---|
936 | path = self.path.abspath(path) |
---|
937 | def command(self, path): |
---|
938 | """Callback function.""" |
---|
939 | with ftputil.error.ftplib_error_to_ftp_os_error: |
---|
940 | self._session.voidcmd("SITE CHMOD 0{0:o} {1}". |
---|
941 | format(mode, path)) |
---|
942 | self._robust_ftp_command(command, path) |
---|
943 | self.stat_cache.invalidate(path) |
---|
944 | |
---|
945 | def __getstate__(self): |
---|
946 | raise TypeError("cannot serialize FTPHost object") |
---|
947 | |
---|
948 | # |
---|
949 | # Context manager methods |
---|
950 | # |
---|
951 | def __enter__(self): |
---|
952 | # Return `self`, so it can be accessed as the variable |
---|
953 | # component of the `with` statement. |
---|
954 | return self |
---|
955 | |
---|
956 | def __exit__(self, exc_type, exc_val, exc_tb): |
---|
957 | # We don't need the `exc_*` arguments here. |
---|
958 | # pylint: disable=unused-argument |
---|
959 | self.close() |
---|
960 | # Be explicit. |
---|
961 | return False |
---|