1 | # Copyright (C) 2002-2015, Stefan Schwarzer <sschwarzer@sschwarzer.net> |
---|
2 | # and ftputil contributors (see `doc/contributors.txt`) |
---|
3 | # See the file LICENSE for licensing terms. |
---|
4 | |
---|
5 | """ |
---|
6 | `FTPHost` is the central class of the `ftputil` library. |
---|
7 | |
---|
8 | See `__init__.py` for an example. |
---|
9 | """ |
---|
10 | |
---|
11 | from __future__ import absolute_import |
---|
12 | from __future__ import print_function |
---|
13 | from __future__ import unicode_literals |
---|
14 | |
---|
15 | import ftplib |
---|
16 | import stat |
---|
17 | import sys |
---|
18 | import time |
---|
19 | import warnings |
---|
20 | |
---|
21 | import ftputil.error |
---|
22 | import ftputil.file |
---|
23 | import ftputil.file_transfer |
---|
24 | import ftputil.path |
---|
25 | import ftputil.session_adapter |
---|
26 | import ftputil.stat |
---|
27 | import ftputil.tool |
---|
28 | |
---|
29 | __all__ = ["FTPHost"] |
---|
30 | |
---|
31 | |
---|
32 | # The "protected" attributes PyLint talks about aren't intended for |
---|
33 | # clients of the library. `FTPHost` objects need to use some of these |
---|
34 | # library-internal attributes though. |
---|
35 | # pylint: disable=protected-access |
---|
36 | |
---|
37 | |
---|
38 | ##################################################################### |
---|
39 | # `FTPHost` class with several methods similar to those of `os` |
---|
40 | |
---|
41 | class FTPHost(object): |
---|
42 | """FTP host class.""" |
---|
43 | |
---|
44 | # Implementation notes: |
---|
45 | # |
---|
46 | # Upon every request of a file (`FTPFile` object) a new FTP |
---|
47 | # session is created (or reused from a cache), leading to a child |
---|
48 | # session of the `FTPHost` object from which the file is requested. |
---|
49 | # |
---|
50 | # This is needed because opening an `FTPFile` will make the |
---|
51 | # local session object wait for the completion of the transfer. |
---|
52 | # In fact, code like this would block indefinitely, if the `RETR` |
---|
53 | # request would be made on the `_session` of the object host: |
---|
54 | # |
---|
55 | # host = FTPHost(ftp_server, user, password) |
---|
56 | # f = host.open("index.html") |
---|
57 | # host.getcwd() # would block! |
---|
58 | # |
---|
59 | # On the other hand, the initially constructed host object will |
---|
60 | # store references to already established `FTPFile` objects and |
---|
61 | # reuse an associated connection if its associated `FTPFile` |
---|
62 | # has been closed. |
---|
63 | |
---|
64 | def __init__(self, *args, **kwargs): |
---|
65 | """Abstract initialization of `FTPHost` object.""" |
---|
66 | # Store arguments for later operations. |
---|
67 | self._args = args |
---|
68 | self._kwargs = kwargs |
---|
69 | #XXX Maybe put the following in a `reset` method. |
---|
70 | # The time shift setting shouldn't be reset though. |
---|
71 | # Make a session according to these arguments. |
---|
72 | self._session = self._make_session() |
---|
73 | # Simulate `os.path`. |
---|
74 | self.path = ftputil.path._Path(self) |
---|
75 | # lstat, stat, listdir services. |
---|
76 | self._stat = ftputil.stat._Stat(self) |
---|
77 | self.stat_cache = self._stat._lstat_cache |
---|
78 | self.stat_cache.enable() |
---|
79 | with ftputil.error.ftplib_error_to_ftp_os_error: |
---|
80 | self._cached_current_dir = \ |
---|
81 | self.path.normpath(ftputil.tool.as_unicode(self._session.pwd())) |
---|
82 | # Associated `FTPHost` objects for data transfer. |
---|
83 | self._children = [] |
---|
84 | # This is only set to something else than `None` if this |
---|
85 | # instance represents an `FTPFile`. |
---|
86 | self._file = None |
---|
87 | # Now opened. |
---|
88 | self.closed = False |
---|
89 | # Set curdir, pardir etc. for the remote host. RFC 959 states |
---|
90 | # that this is, strictly speaking, dependent on the server OS |
---|
91 | # but it seems to work at least with Unix and Windows servers. |
---|
92 | self.curdir, self.pardir, self.sep = ".", "..", "/" |
---|
93 | # Set default time shift (used in `upload_if_newer` and |
---|
94 | # `download_if_newer`). |
---|
95 | self.set_time_shift(0.0) |
---|
96 | # Use `LIST -a` option by default. If this causes problems, |
---|
97 | # the user can set the attribute to `False`. |
---|
98 | warnings.warn( |
---|
99 | "`use_list_a_option` will default to `False` in ftputil 4.x.x", |
---|
100 | DeprecationWarning, stacklevel=2) |
---|
101 | self.use_list_a_option = True |
---|
102 | |
---|
103 | def keep_alive(self): |
---|
104 | """ |
---|
105 | Try to keep the connection alive in order to avoid server timeouts. |
---|
106 | |
---|
107 | Note that this won't help if the connection has already timed |
---|
108 | out! In this case, `keep_alive` will raise an `TemporaryError`. |
---|
109 | (Actually, if you get a server timeout, the error - for a specific |
---|
110 | connection - will be permanent.) |
---|
111 | """ |
---|
112 | # Warning: Don't call this method on `FTPHost` instances which |
---|
113 | # represent file transfers. This may fail in confusing ways. |
---|
114 | with ftputil.error.ftplib_error_to_ftp_os_error: |
---|
115 | # Ignore return value. |
---|
116 | self._session.pwd() |
---|
117 | |
---|
118 | # |
---|
119 | # Dealing with child sessions and file-like objects |
---|
120 | # (rather low-level) |
---|
121 | # |
---|
122 | def _make_session(self): |
---|
123 | """ |
---|
124 | Return a new session object according to the current state of |
---|
125 | this `FTPHost` instance. |
---|
126 | """ |
---|
127 | # Don't modify original attributes below. |
---|
128 | args = self._args[:] |
---|
129 | kwargs = self._kwargs.copy() |
---|
130 | # If a session factory has been given on the instantiation of |
---|
131 | # this `FTPHost` object, use the same factory for this |
---|
132 | # `FTPHost` object's child sessions. |
---|
133 | factory = kwargs.pop("session_factory", ftplib.FTP) |
---|
134 | with ftputil.error.ftplib_error_to_ftp_os_error: |
---|
135 | session = factory(*args, **kwargs) |
---|
136 | # Adapt session so that they accept unicode strings with |
---|
137 | # non-ASCII characters (as long as the string contains only |
---|
138 | # code points <= 255). See the docstring in module |
---|
139 | # `session_adapter` for details. |
---|
140 | if ftputil.compat.python_version == 2: |
---|
141 | session = ftputil.session_adapter.SessionAdapter(session) |
---|
142 | return session |
---|
143 | |
---|
144 | def _copy(self): |
---|
145 | """Return a copy of this `FTPHost` object.""" |
---|
146 | # The copy includes a new session factory return value (aka |
---|
147 | # session) but doesn't copy the state of `self.getcwd()`. |
---|
148 | return self.__class__(*self._args, **self._kwargs) |
---|
149 | |
---|
150 | def _available_child(self): |
---|
151 | """ |
---|
152 | Return an available (i. e. one whose `_file` object is closed |
---|
153 | and doesn't have a timed-out server connection) child |
---|
154 | (`FTPHost` object) from the pool of children or `None` if |
---|
155 | there aren't any. |
---|
156 | """ |
---|
157 | #TODO: Currently timed-out child sessions aren't removed and |
---|
158 | # may collect over time. In very busy or long running |
---|
159 | # processes, this might slow down an application because the |
---|
160 | # same stale child sessions have to be processed again and |
---|
161 | # again. |
---|
162 | for host in self._children: |
---|
163 | # Test for timeouts only after testing for a closed file: |
---|
164 | # - If a file isn't closed, save time; don't bother to access |
---|
165 | # the remote server. |
---|
166 | # - If a file transfer on the child is in progress, requesting |
---|
167 | # the directory is an invalid operation because of the way |
---|
168 | # the FTP state machine works (see RFC 959). |
---|
169 | if host._file.closed: |
---|
170 | try: |
---|
171 | host._session.pwd() |
---|
172 | # Under high load, a 226 status response from a |
---|
173 | # previous download may arrive too late, so that it's |
---|
174 | # "seen" in the `pwd` call. For now, skip the |
---|
175 | # potential child session; it will be considered again |
---|
176 | # when `_available_child` is called the next time. |
---|
177 | except ftplib.error_reply: |
---|
178 | continue |
---|
179 | # Timed-out sessions raise `error_temp`. |
---|
180 | except ftplib.error_temp: |
---|
181 | continue |
---|
182 | # The server may have closed the connection which may |
---|
183 | # cause `host._session.getline` to raise an `EOFError` |
---|
184 | # (see ticket #114). |
---|
185 | except EOFError: |
---|
186 | continue |
---|
187 | # Under high load, there may be a socket read timeout |
---|
188 | # during the last FTP file `close` (see ticket #112). |
---|
189 | # Note that a socket timeout is quite different from |
---|
190 | # an FTP session timeout. |
---|
191 | except OSError: |
---|
192 | continue |
---|
193 | else: |
---|
194 | # Everything's ok; use this `FTPHost` instance. |
---|
195 | return host |
---|
196 | # Be explicit. |
---|
197 | return None |
---|
198 | |
---|
199 | def open(self, path, mode="r", buffering=None, encoding=None, errors=None, |
---|
200 | newline=None, rest=None): |
---|
201 | """ |
---|
202 | Return an open file(-like) object which is associated with |
---|
203 | this `FTPHost` object. |
---|
204 | |
---|
205 | The arguments `path`, `mode`, `buffering`, `encoding`, `errors` |
---|
206 | and `newlines` have the same meaning as for `io.open`. If `rest` |
---|
207 | is given as an integer, |
---|
208 | |
---|
209 | - reading will start at the byte (zero-based) `rest` |
---|
210 | - writing will overwrite the remote file from byte `rest` |
---|
211 | |
---|
212 | This method tries to reuse a child but will generate a new one |
---|
213 | if none is available. |
---|
214 | """ |
---|
215 | # Support the same arguments as `io.open`. |
---|
216 | # pylint: disable=too-many-arguments |
---|
217 | path = ftputil.tool.as_unicode(path) |
---|
218 | host = self._available_child() |
---|
219 | if host is None: |
---|
220 | host = self._copy() |
---|
221 | self._children.append(host) |
---|
222 | host._file = ftputil.file.FTPFile(host) |
---|
223 | basedir = self.getcwd() |
---|
224 | # Prepare for changing the directory (see whitespace workaround |
---|
225 | # in method `_dir`). |
---|
226 | if host.path.isabs(path): |
---|
227 | effective_path = path |
---|
228 | else: |
---|
229 | effective_path = host.path.join(basedir, path) |
---|
230 | effective_dir, effective_file = host.path.split(effective_path) |
---|
231 | try: |
---|
232 | # This will fail if the directory isn't accessible at all. |
---|
233 | host.chdir(effective_dir) |
---|
234 | except ftputil.error.PermanentError: |
---|
235 | # Similarly to a failed `file` in a local file system, |
---|
236 | # raise an `IOError`, not an `OSError`. |
---|
237 | raise ftputil.error.FTPIOError("remote directory '{0}' doesn't " |
---|
238 | "exist or has insufficient access rights". |
---|
239 | format(effective_dir)) |
---|
240 | host._file._open(effective_file, mode=mode, buffering=buffering, |
---|
241 | encoding=encoding, errors=errors, newline=newline, |
---|
242 | rest=rest) |
---|
243 | if "w" in mode: |
---|
244 | # Invalidate cache entry because size and timestamps will change. |
---|
245 | self.stat_cache.invalidate(effective_path) |
---|
246 | return host._file |
---|
247 | |
---|
248 | def close(self): |
---|
249 | """Close host connection.""" |
---|
250 | if self.closed: |
---|
251 | return |
---|
252 | # Close associated children. |
---|
253 | for host in self._children: |
---|
254 | # Children have a `_file` attribute which is an `FTPFile` object. |
---|
255 | host._file.close() |
---|
256 | host.close() |
---|
257 | # Now deal with ourself. |
---|
258 | try: |
---|
259 | with ftputil.error.ftplib_error_to_ftp_os_error: |
---|
260 | self._session.close() |
---|
261 | finally: |
---|
262 | # If something went wrong before, the host/session is |
---|
263 | # probably defunct and subsequent calls to `close` won't |
---|
264 | # help either, so consider the host/session closed for |
---|
265 | # practical purposes. |
---|
266 | self.stat_cache.clear() |
---|
267 | self._children = [] |
---|
268 | self.closed = True |
---|
269 | |
---|
270 | # |
---|
271 | # Setting a custom directory parser |
---|
272 | # |
---|
273 | def set_parser(self, parser): |
---|
274 | """ |
---|
275 | Set the parser for extracting stat results from directory |
---|
276 | listings. |
---|
277 | |
---|
278 | The parser interface is described in the documentation, but |
---|
279 | here are the most important things: |
---|
280 | |
---|
281 | - A parser should derive from `ftputil.stat.Parser`. |
---|
282 | |
---|
283 | - The parser has to implement two methods, `parse_line` and |
---|
284 | `ignores_line`. For the latter, there's a probably useful |
---|
285 | default in the class `ftputil.stat.Parser`. |
---|
286 | |
---|
287 | - `parse_line` should try to parse a line of a directory |
---|
288 | listing and return a `ftputil.stat.StatResult` instance. If |
---|
289 | parsing isn't possible, raise `ftputil.error.ParserError` |
---|
290 | with a useful error message. |
---|
291 | |
---|
292 | - `ignores_line` should return a true value if the line isn't |
---|
293 | assumed to contain stat information. |
---|
294 | """ |
---|
295 | # The cache contents, if any, probably aren't useful. |
---|
296 | self.stat_cache.clear() |
---|
297 | # Set the parser explicitly, don't allow "smart" switching anymore. |
---|
298 | self._stat._parser = parser |
---|
299 | self._stat._allow_parser_switching = False |
---|
300 | |
---|
301 | # |
---|
302 | # Time shift adjustment between client (i. e. us) and server |
---|
303 | # |
---|
304 | def set_time_shift(self, time_shift): |
---|
305 | """ |
---|
306 | Set the time shift value (i. e. the time difference between |
---|
307 | client and server) for this `FTPHost` object. By (my) |
---|
308 | definition, the time shift value is positive if the local |
---|
309 | time of the server is greater than the local time of the |
---|
310 | client (for the same physical time), i. e. |
---|
311 | |
---|
312 | time_shift =def= t_server - t_client |
---|
313 | <=> t_server = t_client + time_shift |
---|
314 | <=> t_client = t_server - time_shift |
---|
315 | |
---|
316 | The time shift is measured in seconds. |
---|
317 | """ |
---|
318 | # Implicitly set via `set_time_shift` call in constructor |
---|
319 | # pylint: disable=attribute-defined-outside-init |
---|
320 | self._time_shift = time_shift |
---|
321 | |
---|
322 | def time_shift(self): |
---|
323 | """ |
---|
324 | Return the time shift between FTP server and client. See the |
---|
325 | docstring of `set_time_shift` for more on this value. |
---|
326 | """ |
---|
327 | return self._time_shift |
---|
328 | |
---|
329 | @staticmethod |
---|
330 | def __rounded_time_shift(time_shift): |
---|
331 | """ |
---|
332 | Return the given time shift in seconds, but rounded to |
---|
333 | full hours. The argument is also assumed to be given in |
---|
334 | seconds. |
---|
335 | """ |
---|
336 | minute = 60.0 |
---|
337 | hour = 60.0 * minute |
---|
338 | # Avoid division by zero below. |
---|
339 | if time_shift == 0: |
---|
340 | return 0.0 |
---|
341 | # Use a positive value for rounding. |
---|
342 | absolute_time_shift = abs(time_shift) |
---|
343 | signum = time_shift / absolute_time_shift |
---|
344 | # Round absolute time shift to 15-minute units. |
---|
345 | absolute_rounded_time_shift = ( |
---|
346 | int( (absolute_time_shift + (7.5*minute)) / (15.0*minute) ) * |
---|
347 | (15.0*minute)) |
---|
348 | # Return with correct sign. |
---|
349 | return signum * absolute_rounded_time_shift |
---|
350 | |
---|
351 | def __assert_valid_time_shift(self, time_shift): |
---|
352 | """ |
---|
353 | Perform sanity checks on the time shift value (given in |
---|
354 | seconds). If the value is invalid, raise a `TimeShiftError`, |
---|
355 | else simply return `None`. |
---|
356 | """ |
---|
357 | minute = 60.0 # seconds |
---|
358 | hour = 60.0 * minute |
---|
359 | absolute_rounded_time_shift = \ |
---|
360 | abs(self.__rounded_time_shift(time_shift)) |
---|
361 | # Test 1: Fail if the absolute time shift is greater than |
---|
362 | # a full day (24 hours). |
---|
363 | if absolute_rounded_time_shift > 24 * hour: |
---|
364 | raise ftputil.error.TimeShiftError( |
---|
365 | "time shift abs({0:.2f} s) > 1 day".format(time_shift)) |
---|
366 | # Test 2: Fail if the deviation between given time shift and |
---|
367 | # 15-minute units is greater than a certain limit. |
---|
368 | maximum_deviation = 5 * minute |
---|
369 | if abs(time_shift - self.__rounded_time_shift(time_shift)) > \ |
---|
370 | maximum_deviation: |
---|
371 | raise ftputil.error.TimeShiftError( |
---|
372 | "time shift ({0:.2f} s) deviates more than {1:d} s " |
---|
373 | "from 15-minute units".format( |
---|
374 | time_shift, int(maximum_deviation))) |
---|
375 | |
---|
376 | def synchronize_times(self): |
---|
377 | """ |
---|
378 | Synchronize the local times of FTP client and server. This |
---|
379 | is necessary to let `upload_if_newer` and `download_if_newer` |
---|
380 | work correctly. If `synchronize_times` isn't applicable |
---|
381 | (see below), the time shift can still be set explicitly with |
---|
382 | `set_time_shift`. |
---|
383 | |
---|
384 | This implementation of `synchronize_times` requires _all_ of |
---|
385 | the following: |
---|
386 | |
---|
387 | - The connection between server and client is established. |
---|
388 | - The client has write access to the directory that is |
---|
389 | current when `synchronize_times` is called. |
---|
390 | |
---|
391 | The common usage pattern of `synchronize_times` is to call it |
---|
392 | directly after the connection is established. (As can be |
---|
393 | concluded from the points above, this requires write access |
---|
394 | to the login directory.) |
---|
395 | |
---|
396 | If `synchronize_times` fails, it raises a `TimeShiftError`. |
---|
397 | """ |
---|
398 | helper_file_name = "_ftputil_sync_" |
---|
399 | # Open a dummy file for writing in the current directory |
---|
400 | # on the FTP host, then close it. |
---|
401 | try: |
---|
402 | # May raise `FTPIOError` if directory isn't writable. |
---|
403 | file_ = self.open(helper_file_name, "w") |
---|
404 | file_.close() |
---|
405 | except ftputil.error.FTPIOError: |
---|
406 | raise ftputil.error.TimeShiftError( |
---|
407 | """couldn't write helper file in directory '{0}'""". |
---|
408 | format(self.getcwd())) |
---|
409 | # If everything worked up to here it should be possible to stat |
---|
410 | # and then remove the just-written file. |
---|
411 | try: |
---|
412 | server_time = self.path.getmtime(helper_file_name) |
---|
413 | self.unlink(helper_file_name) |
---|
414 | except ftputil.error.FTPOSError: |
---|
415 | # If we got a `TimeShiftError` exception above, we should't |
---|
416 | # come here: if we did not get a `TimeShiftError` above, |
---|
417 | # deletion should be possible. The only reason for an exception |
---|
418 | # I can think of here is a race condition by removing write |
---|
419 | # permission from the directory or helper file after it has been |
---|
420 | # written to. |
---|
421 | raise ftputil.error.TimeShiftError( |
---|
422 | "could write helper file but not unlink it") |
---|
423 | # Calculate the difference between server and client. |
---|
424 | now = time.time() |
---|
425 | time_shift = server_time - now |
---|
426 | # As the time shift for this host instance isn't set yet, the |
---|
427 | # directory parser will calculate times one year in the past if |
---|
428 | # the time zone of the server is east from ours. Thus the time |
---|
429 | # shift will be off by a year as well (see ticket #55). |
---|
430 | if time_shift < -360 * 24 * 60 * 60: |
---|
431 | # Re-add one year and re-calculate the time shift. We don't |
---|
432 | # know how many days made up that year (it might have been |
---|
433 | # a leap year), so go the route via `time.localtime` and |
---|
434 | # `time.mktime`. |
---|
435 | server_time_struct = time.localtime(server_time) |
---|
436 | server_time_struct = (server_time_struct.tm_year+1,) + \ |
---|
437 | server_time_struct[1:] |
---|
438 | server_time = time.mktime(server_time_struct) |
---|
439 | time_shift = server_time - now |
---|
440 | # Do some sanity checks. |
---|
441 | self.__assert_valid_time_shift(time_shift) |
---|
442 | # If tests passed, store the time difference as time shift value. |
---|
443 | self.set_time_shift(self.__rounded_time_shift(time_shift)) |
---|
444 | |
---|
445 | # |
---|
446 | # Operations based on file-like objects (rather high-level), |
---|
447 | # like upload and download |
---|
448 | # |
---|
449 | # This code doesn't complain if the chunk size is passed as a |
---|
450 | # positional argument but emits a deprecation warning if `length` |
---|
451 | # is used as a keyword argument. |
---|
452 | @staticmethod |
---|
453 | def copyfileobj(source, target, |
---|
454 | max_chunk_size=ftputil.file_transfer.MAX_COPY_CHUNK_SIZE, |
---|
455 | callback=None): |
---|
456 | """ |
---|
457 | Copy data from file-like object `source` to file-like object |
---|
458 | `target`. |
---|
459 | """ |
---|
460 | ftputil.file_transfer.copyfileobj(source, target, max_chunk_size, |
---|
461 | callback) |
---|
462 | |
---|
463 | def _upload_files(self, source_path, target_path): |
---|
464 | """ |
---|
465 | Return a `LocalFile` and `RemoteFile` as source and target, |
---|
466 | respectively. |
---|
467 | |
---|
468 | The strings `source_path` and `target_path` are the (absolute |
---|
469 | or relative) paths of the local and the remote file, respectively. |
---|
470 | """ |
---|
471 | source_file = ftputil.file_transfer.LocalFile(source_path, "rb") |
---|
472 | # Passing `self` (the `FTPHost` instance) here is correct. |
---|
473 | target_file = ftputil.file_transfer.RemoteFile(self, target_path, "wb") |
---|
474 | return source_file, target_file |
---|
475 | |
---|
476 | def upload(self, source, target, callback=None): |
---|
477 | """ |
---|
478 | Upload a file from the local source (name) to the remote |
---|
479 | target (name). |
---|
480 | |
---|
481 | If a callable `callback` is given, it's called after every |
---|
482 | chunk of transferred data. The chunk size is a constant |
---|
483 | defined in `file_transfer`. The callback will be called with a |
---|
484 | single argument, the data chunk that was transferred before |
---|
485 | the callback was called. |
---|
486 | """ |
---|
487 | target = ftputil.tool.as_unicode(target) |
---|
488 | source_file, target_file = self._upload_files(source, target) |
---|
489 | ftputil.file_transfer.copy_file(source_file, target_file, |
---|
490 | conditional=False, callback=callback) |
---|
491 | |
---|
492 | def upload_if_newer(self, source, target, callback=None): |
---|
493 | """ |
---|
494 | Upload a file only if it's newer than the target on the |
---|
495 | remote host or if the target file does not exist. See the |
---|
496 | method `upload` for the meaning of the parameters. |
---|
497 | |
---|
498 | If an upload was necessary, return `True`, else return `False`. |
---|
499 | |
---|
500 | If a callable `callback` is given, it's called after every |
---|
501 | chunk of transferred data. The chunk size is a constant |
---|
502 | defined in `file_transfer`. The callback will be called with a |
---|
503 | single argument, the data chunk that was transferred before |
---|
504 | the callback was called. |
---|
505 | """ |
---|
506 | target = ftputil.tool.as_unicode(target) |
---|
507 | source_file, target_file = self._upload_files(source, target) |
---|
508 | return ftputil.file_transfer.copy_file(source_file, target_file, |
---|
509 | conditional=True, |
---|
510 | callback=callback) |
---|
511 | |
---|
512 | def _download_files(self, source_path, target_path): |
---|
513 | """ |
---|
514 | Return a `RemoteFile` and `LocalFile` as source and target, |
---|
515 | respectively. |
---|
516 | |
---|
517 | The strings `source_path` and `target_path` are the (absolute |
---|
518 | or relative) paths of the remote and the local file, respectively. |
---|
519 | """ |
---|
520 | source_file = ftputil.file_transfer.RemoteFile(self, source_path, "rb") |
---|
521 | target_file = ftputil.file_transfer.LocalFile(target_path, "wb") |
---|
522 | return source_file, target_file |
---|
523 | |
---|
524 | def download(self, source, target, callback=None): |
---|
525 | """ |
---|
526 | Download a file from the remote source (name) to the local |
---|
527 | target (name). |
---|
528 | |
---|
529 | If a callable `callback` is given, it's called after every |
---|
530 | chunk of transferred data. The chunk size is a constant |
---|
531 | defined in `file_transfer`. The callback will be called with a |
---|
532 | single argument, the data chunk that was transferred before |
---|
533 | the callback was called. |
---|
534 | """ |
---|
535 | source = ftputil.tool.as_unicode(source) |
---|
536 | source_file, target_file = self._download_files(source, target) |
---|
537 | ftputil.file_transfer.copy_file(source_file, target_file, |
---|
538 | conditional=False, callback=callback) |
---|
539 | |
---|
540 | def download_if_newer(self, source, target, callback=None): |
---|
541 | """ |
---|
542 | Download a file only if it's newer than the target on the |
---|
543 | local host or if the target file does not exist. See the |
---|
544 | method `download` for the meaning of the parameters. |
---|
545 | |
---|
546 | If a download was necessary, return `True`, else return |
---|
547 | `False`. |
---|
548 | |
---|
549 | If a callable `callback` is given, it's called after every |
---|
550 | chunk of transferred data. The chunk size is a constant |
---|
551 | defined in `file_transfer`. The callback will be called with a |
---|
552 | single argument, the data chunk that was transferred before |
---|
553 | the callback was called. |
---|
554 | """ |
---|
555 | source = ftputil.tool.as_unicode(source) |
---|
556 | source_file, target_file = self._download_files(source, target) |
---|
557 | return ftputil.file_transfer.copy_file(source_file, target_file, |
---|
558 | conditional=True, |
---|
559 | callback=callback) |
---|
560 | |
---|
561 | # |
---|
562 | # Helper methods to descend into a directory before executing a command |
---|
563 | # |
---|
564 | def _check_inaccessible_login_directory(self): |
---|
565 | """ |
---|
566 | Raise an `InaccessibleLoginDirError` exception if we can't |
---|
567 | change to the login directory. This test is only reliable if |
---|
568 | the current directory is the login directory. |
---|
569 | """ |
---|
570 | presumable_login_dir = self.getcwd() |
---|
571 | # Bail out with an internal error rather than modify the |
---|
572 | # current directory without hope of restoration. |
---|
573 | try: |
---|
574 | self.chdir(presumable_login_dir) |
---|
575 | except ftputil.error.PermanentError: |
---|
576 | raise ftputil.error.InaccessibleLoginDirError( |
---|
577 | "directory '{0}' is not accessible". |
---|
578 | format(presumable_login_dir)) |
---|
579 | |
---|
580 | def _robust_ftp_command(self, command, path, descend_deeply=False): |
---|
581 | """ |
---|
582 | Run an FTP command on a path. The return value of the method |
---|
583 | is the return value of the command. |
---|
584 | |
---|
585 | If `descend_deeply` is true (the default is false), descend |
---|
586 | deeply, i. e. change the directory to the end of the path. |
---|
587 | """ |
---|
588 | # If we can't change to the yet-current directory, the code |
---|
589 | # below won't work (see below), so in this case rather raise |
---|
590 | # an exception than giving wrong results. |
---|
591 | self._check_inaccessible_login_directory() |
---|
592 | # Some FTP servers don't behave as expected if the directory |
---|
593 | # portion of the path contains whitespace; some even yield |
---|
594 | # strange results if the command isn't executed in the |
---|
595 | # current directory. Therefore, change to the directory |
---|
596 | # which contains the item to run the command on and invoke |
---|
597 | # the command just there. |
---|
598 | # |
---|
599 | # Remember old working directory. |
---|
600 | old_dir = self.getcwd() |
---|
601 | try: |
---|
602 | if descend_deeply: |
---|
603 | # Invoke the command in (not: on) the deepest directory. |
---|
604 | self.chdir(path) |
---|
605 | # Workaround for some servers that give recursive |
---|
606 | # listings when called with a dot as path; see issue #33, |
---|
607 | # http://ftputil.sschwarzer.net/trac/ticket/33 |
---|
608 | return command(self, "") |
---|
609 | else: |
---|
610 | # Invoke the command in the "next to last" directory. |
---|
611 | head, tail = self.path.split(path) |
---|
612 | self.chdir(head) |
---|
613 | return command(self, tail) |
---|
614 | finally: |
---|
615 | self.chdir(old_dir) |
---|
616 | |
---|
617 | # |
---|
618 | # Miscellaneous utility methods resembling functions in `os` |
---|
619 | # |
---|
620 | def getcwd(self): |
---|
621 | """Return the current path name.""" |
---|
622 | return self._cached_current_dir |
---|
623 | |
---|
624 | def chdir(self, path): |
---|
625 | """Change the directory on the host.""" |
---|
626 | path = ftputil.tool.as_unicode(path) |
---|
627 | with ftputil.error.ftplib_error_to_ftp_os_error: |
---|
628 | self._session.cwd(path) |
---|
629 | # The path given as the argument is relative to the old current |
---|
630 | # directory, therefore join them. |
---|
631 | self._cached_current_dir = \ |
---|
632 | self.path.normpath(self.path.join(self._cached_current_dir, path)) |
---|
633 | |
---|
634 | # Ignore unused argument `mode` |
---|
635 | # pylint: disable=unused-argument |
---|
636 | def mkdir(self, path, mode=None): |
---|
637 | """ |
---|
638 | Make the directory path on the remote host. The argument |
---|
639 | `mode` is ignored and only "supported" for similarity with |
---|
640 | `os.mkdir`. |
---|
641 | """ |
---|
642 | path = ftputil.tool.as_unicode(path) |
---|
643 | def command(self, path): |
---|
644 | """Callback function.""" |
---|
645 | with ftputil.error.ftplib_error_to_ftp_os_error: |
---|
646 | self._session.mkd(path) |
---|
647 | self._robust_ftp_command(command, path) |
---|
648 | |
---|
649 | # TODO: The virtual directory support doesn't have unit tests yet |
---|
650 | # because the mocking most likely would be quite complicated. The |
---|
651 | # tests should be added when mainly the `mock` library is used |
---|
652 | # instead of the mock code in `test.mock_ftplib`. |
---|
653 | # |
---|
654 | # Ignore unused argument `mode` |
---|
655 | # pylint: disable=unused-argument |
---|
656 | def makedirs(self, path, mode=None): |
---|
657 | """ |
---|
658 | Make the directory `path`, but also make not yet existing |
---|
659 | intermediate directories, like `os.makedirs`. The value of |
---|
660 | `mode` is only accepted for compatibility with `os.makedirs` |
---|
661 | but otherwise ignored. |
---|
662 | """ |
---|
663 | path = ftputil.tool.as_unicode(path) |
---|
664 | path = self.path.abspath(path) |
---|
665 | directories = path.split(self.sep) |
---|
666 | old_dir = self.getcwd() |
---|
667 | try: |
---|
668 | # Try to build the directory chain from the "uppermost" to |
---|
669 | # the "lowermost" directory. |
---|
670 | for index in range(1, len(directories)): |
---|
671 | # Re-insert the separator which got lost by using |
---|
672 | # `path.split`. |
---|
673 | next_directory = (self.sep + |
---|
674 | self.path.join(*directories[:index+1])) |
---|
675 | # If we have "virtual directories" (see #86), just |
---|
676 | # listing the parent directory won't tell us if a |
---|
677 | # directory actually exists. So try to change into the |
---|
678 | # directory. |
---|
679 | try: |
---|
680 | self.chdir(next_directory) |
---|
681 | except ftputil.error.PermanentError: |
---|
682 | try: |
---|
683 | self.mkdir(next_directory) |
---|
684 | except ftputil.error.PermanentError: |
---|
685 | # Find out the cause of the error. Re-raise |
---|
686 | # the exception only if the directory didn't |
---|
687 | # exist already, else something went _really_ |
---|
688 | # wrong, e. g. there's a regular file with the |
---|
689 | # name of the directory. |
---|
690 | if not self.path.isdir(next_directory): |
---|
691 | raise |
---|
692 | finally: |
---|
693 | self.chdir(old_dir) |
---|
694 | |
---|
695 | def rmdir(self, path): |
---|
696 | """ |
---|
697 | Remove the _empty_ directory `path` on the remote host. |
---|
698 | |
---|
699 | Compatibility note: |
---|
700 | |
---|
701 | Previous versions of ftputil could possibly delete non- |
---|
702 | empty directories as well, - if the server allowed it. This |
---|
703 | is no longer supported. |
---|
704 | """ |
---|
705 | path = ftputil.tool.as_unicode(path) |
---|
706 | path = self.path.abspath(path) |
---|
707 | if self.listdir(path): |
---|
708 | raise ftputil.error.PermanentError("directory '{0}' not empty". |
---|
709 | format(path)) |
---|
710 | #XXX How does `rmd` work with links? |
---|
711 | def command(self, path): |
---|
712 | """Callback function.""" |
---|
713 | with ftputil.error.ftplib_error_to_ftp_os_error: |
---|
714 | self._session.rmd(path) |
---|
715 | self._robust_ftp_command(command, path) |
---|
716 | self.stat_cache.invalidate(path) |
---|
717 | |
---|
718 | def remove(self, path): |
---|
719 | """ |
---|
720 | Remove the file or link given by `path`. |
---|
721 | |
---|
722 | Raise a `PermanentError` if the path doesn't exist, but maybe |
---|
723 | raise other exceptions depending on the state of the server |
---|
724 | (e. g. timeout). |
---|
725 | """ |
---|
726 | path = ftputil.tool.as_unicode(path) |
---|
727 | path = self.path.abspath(path) |
---|
728 | # Though `isfile` includes also links to files, `islink` |
---|
729 | # is needed to include links to directories. |
---|
730 | if self.path.isfile(path) or self.path.islink(path) or \ |
---|
731 | not self.path.exists(path): |
---|
732 | # If the path doesn't exist, let the removal command trigger |
---|
733 | # an exception with a more appropriate error message. |
---|
734 | def command(self, path): |
---|
735 | """Callback function.""" |
---|
736 | with ftputil.error.ftplib_error_to_ftp_os_error: |
---|
737 | self._session.delete(path) |
---|
738 | self._robust_ftp_command(command, path) |
---|
739 | else: |
---|
740 | raise ftputil.error.PermanentError( |
---|
741 | "remove/unlink can only delete files and links, " |
---|
742 | "not directories") |
---|
743 | self.stat_cache.invalidate(path) |
---|
744 | |
---|
745 | unlink = remove |
---|
746 | |
---|
747 | def rmtree(self, path, ignore_errors=False, onerror=None): |
---|
748 | """ |
---|
749 | Remove the given remote, possibly non-empty, directory tree. |
---|
750 | The interface of this method is rather complex, in favor of |
---|
751 | compatibility with `shutil.rmtree`. |
---|
752 | |
---|
753 | If `ignore_errors` is set to a true value, errors are ignored. |
---|
754 | If `ignore_errors` is a false value _and_ `onerror` isn't set, |
---|
755 | all exceptions occurring during the tree iteration and |
---|
756 | processing are raised. These exceptions are all of type |
---|
757 | `PermanentError`. |
---|
758 | |
---|
759 | To distinguish between error situations, pass in a callable |
---|
760 | for `onerror`. This callable must accept three arguments: |
---|
761 | `func`, `path` and `exc_info`. `func` is a bound method |
---|
762 | object, _for example_ `your_host_object.listdir`. `path` is |
---|
763 | the path that was the recent argument of the respective method |
---|
764 | (`listdir`, `remove`, `rmdir`). `exc_info` is the exception |
---|
765 | info as it's got from `sys.exc_info`. |
---|
766 | |
---|
767 | Implementation note: The code is copied from `shutil.rmtree` |
---|
768 | in Python 2.4 and adapted to ftputil. |
---|
769 | """ |
---|
770 | path = ftputil.tool.as_unicode(path) |
---|
771 | # The following code is an adapted version of Python 2.4's |
---|
772 | # `shutil.rmtree` function. |
---|
773 | if ignore_errors: |
---|
774 | def new_onerror(*args): |
---|
775 | """Do nothing.""" |
---|
776 | # pylint: disable=unused-argument |
---|
777 | pass |
---|
778 | elif onerror is None: |
---|
779 | def new_onerror(*args): |
---|
780 | """Re-raise exception.""" |
---|
781 | # pylint: disable=unused-argument |
---|
782 | raise |
---|
783 | else: |
---|
784 | new_onerror = onerror |
---|
785 | names = [] |
---|
786 | try: |
---|
787 | names = self.listdir(path) |
---|
788 | except ftputil.error.PermanentError: |
---|
789 | new_onerror(self.listdir, path, sys.exc_info()) |
---|
790 | for name in names: |
---|
791 | full_name = self.path.join(path, name) |
---|
792 | try: |
---|
793 | mode = self.lstat(full_name).st_mode |
---|
794 | except ftputil.error.PermanentError: |
---|
795 | mode = 0 |
---|
796 | if stat.S_ISDIR(mode): |
---|
797 | self.rmtree(full_name, ignore_errors, new_onerror) |
---|
798 | else: |
---|
799 | try: |
---|
800 | self.remove(full_name) |
---|
801 | except ftputil.error.PermanentError: |
---|
802 | new_onerror(self.remove, full_name, sys.exc_info()) |
---|
803 | try: |
---|
804 | self.rmdir(path) |
---|
805 | except ftputil.error.FTPOSError: |
---|
806 | new_onerror(self.rmdir, path, sys.exc_info()) |
---|
807 | |
---|
808 | def rename(self, source, target): |
---|
809 | """Rename the source on the FTP host to target.""" |
---|
810 | source = ftputil.tool.as_unicode(source) |
---|
811 | target = ftputil.tool.as_unicode(target) |
---|
812 | # The following code is in spirit similar to the code in the |
---|
813 | # method `_robust_ftp_command`, though we do _not_ do |
---|
814 | # _everything_ imaginable. |
---|
815 | self._check_inaccessible_login_directory() |
---|
816 | source_head, source_tail = self.path.split(source) |
---|
817 | target_head, target_tail = self.path.split(target) |
---|
818 | paths_contain_whitespace = (" " in source_head) or (" " in target_head) |
---|
819 | if paths_contain_whitespace and source_head == target_head: |
---|
820 | # Both items are in the same directory. |
---|
821 | old_dir = self.getcwd() |
---|
822 | try: |
---|
823 | self.chdir(source_head) |
---|
824 | with ftputil.error.ftplib_error_to_ftp_os_error: |
---|
825 | self._session.rename(source_tail, target_tail) |
---|
826 | finally: |
---|
827 | self.chdir(old_dir) |
---|
828 | else: |
---|
829 | # Use straightforward command. |
---|
830 | with ftputil.error.ftplib_error_to_ftp_os_error: |
---|
831 | self._session.rename(source, target) |
---|
832 | |
---|
833 | #XXX One could argue to put this method into the `_Stat` class, but |
---|
834 | # I refrained from that because then `_Stat` would have to know |
---|
835 | # about `FTPHost`'s `_session` attribute and in turn about |
---|
836 | # `_session`'s `dir` method. |
---|
837 | def _dir(self, path): |
---|
838 | """Return a directory listing as made by FTP's `LIST` command.""" |
---|
839 | # Don't use `self.path.isdir` in this method because that |
---|
840 | # would cause a call of `(l)stat` and thus a call to `_dir`, |
---|
841 | # so we would end up with an infinite recursion. |
---|
842 | def _FTPHost_dir_command(self, path): |
---|
843 | """Callback function.""" |
---|
844 | lines = [] |
---|
845 | def callback(line): |
---|
846 | """Callback function.""" |
---|
847 | lines.append(ftputil.tool.as_unicode(line)) |
---|
848 | with ftputil.error.ftplib_error_to_ftp_os_error: |
---|
849 | if self.use_list_a_option: |
---|
850 | self._session.dir("-a", path, callback) |
---|
851 | else: |
---|
852 | self._session.dir(path, callback) |
---|
853 | return lines |
---|
854 | lines = self._robust_ftp_command(_FTPHost_dir_command, path, |
---|
855 | descend_deeply=True) |
---|
856 | return lines |
---|
857 | |
---|
858 | # The `listdir`, `lstat` and `stat` methods don't use |
---|
859 | # `_robust_ftp_command` because they implicitly already use |
---|
860 | # `_dir` which actually uses `_robust_ftp_command`. |
---|
861 | def listdir(self, path): |
---|
862 | """ |
---|
863 | Return a list of directories, files etc. in the directory |
---|
864 | named `path`. |
---|
865 | |
---|
866 | If the directory listing from the server can't be parsed with |
---|
867 | any of the available parsers raise a `ParserError`. |
---|
868 | """ |
---|
869 | original_path = path |
---|
870 | path = ftputil.tool.as_unicode(path) |
---|
871 | items = self._stat._listdir(path) |
---|
872 | return [ftputil.tool.same_string_type_as(original_path, item) |
---|
873 | for item in items] |
---|
874 | |
---|
875 | def lstat(self, path, _exception_for_missing_path=True): |
---|
876 | """ |
---|
877 | Return an object similar to that returned by `os.lstat`. |
---|
878 | |
---|
879 | If the directory listing from the server can't be parsed with |
---|
880 | any of the available parsers, raise a `ParserError`. If the |
---|
881 | directory _can_ be parsed and the `path` is _not_ found, raise |
---|
882 | a `PermanentError`. |
---|
883 | |
---|
884 | (`_exception_for_missing_path` is an implementation aid and |
---|
885 | _not_ intended for use by ftputil clients.) |
---|
886 | """ |
---|
887 | path = ftputil.tool.as_unicode(path) |
---|
888 | return self._stat._lstat(path, _exception_for_missing_path) |
---|
889 | |
---|
890 | def stat(self, path, _exception_for_missing_path=True): |
---|
891 | """ |
---|
892 | Return info from a "stat" call on `path`. |
---|
893 | |
---|
894 | If the directory containing `path` can't be parsed, raise a |
---|
895 | `ParserError`. If the directory containing `path` can be |
---|
896 | parsed but the `path` can't be found, raise a |
---|
897 | `PermanentError`. Also raise a `PermanentError` if there's an |
---|
898 | endless (cyclic) chain of symbolic links "behind" the `path`. |
---|
899 | |
---|
900 | (`_exception_for_missing_path` is an implementation aid and |
---|
901 | _not_ intended for use by ftputil clients.) |
---|
902 | """ |
---|
903 | path = ftputil.tool.as_unicode(path) |
---|
904 | return self._stat._stat(path, _exception_for_missing_path) |
---|
905 | |
---|
906 | def walk(self, top, topdown=True, onerror=None, followlinks=False): |
---|
907 | """ |
---|
908 | Iterate over directory tree and return a tuple (dirpath, |
---|
909 | dirnames, filenames) on each iteration, like the `os.walk` |
---|
910 | function (see https://docs.python.org/library/os.html#os.walk ). |
---|
911 | """ |
---|
912 | top = ftputil.tool.as_unicode(top) |
---|
913 | # The following code is copied from `os.walk` in Python 2.4 |
---|
914 | # and adapted to ftputil. |
---|
915 | try: |
---|
916 | names = self.listdir(top) |
---|
917 | except ftputil.error.FTPOSError as err: |
---|
918 | if onerror is not None: |
---|
919 | onerror(err) |
---|
920 | return |
---|
921 | dirs, nondirs = [], [] |
---|
922 | for name in names: |
---|
923 | if self.path.isdir(self.path.join(top, name)): |
---|
924 | dirs.append(name) |
---|
925 | else: |
---|
926 | nondirs.append(name) |
---|
927 | if topdown: |
---|
928 | yield top, dirs, nondirs |
---|
929 | for name in dirs: |
---|
930 | path = self.path.join(top, name) |
---|
931 | if followlinks or not self.path.islink(path): |
---|
932 | for item in self.walk(path, topdown, onerror, followlinks): |
---|
933 | yield item |
---|
934 | if not topdown: |
---|
935 | yield top, dirs, nondirs |
---|
936 | |
---|
937 | def chmod(self, path, mode): |
---|
938 | """ |
---|
939 | Change the mode of a remote `path` (a string) to the integer |
---|
940 | `mode`. This integer uses the same bits as the mode value |
---|
941 | returned by the `stat` and `lstat` commands. |
---|
942 | |
---|
943 | If something goes wrong, raise a `TemporaryError` or a |
---|
944 | `PermanentError`, according to the status code returned by |
---|
945 | the server. In particular, a non-existent path usually |
---|
946 | causes a `PermanentError`. |
---|
947 | """ |
---|
948 | path = ftputil.tool.as_unicode(path) |
---|
949 | path = self.path.abspath(path) |
---|
950 | def command(self, path): |
---|
951 | """Callback function.""" |
---|
952 | with ftputil.error.ftplib_error_to_ftp_os_error: |
---|
953 | self._session.voidcmd("SITE CHMOD 0{0:o} {1}". |
---|
954 | format(mode, path)) |
---|
955 | self._robust_ftp_command(command, path) |
---|
956 | self.stat_cache.invalidate(path) |
---|
957 | |
---|
958 | def __getstate__(self): |
---|
959 | raise TypeError("cannot serialize FTPHost object") |
---|
960 | |
---|
961 | # |
---|
962 | # Context manager methods |
---|
963 | # |
---|
964 | def __enter__(self): |
---|
965 | # Return `self`, so it can be accessed as the variable |
---|
966 | # component of the `with` statement. |
---|
967 | return self |
---|
968 | |
---|
969 | def __exit__(self, exc_type, exc_val, exc_tb): |
---|
970 | # We don't need the `exc_*` arguments here. |
---|
971 | # pylint: disable=unused-argument |
---|
972 | self.close() |
---|
973 | # Be explicit. |
---|
974 | return False |
---|