source: ftputil/host.py @ 1745:2402b8a75178

Last change on this file since 1745:2402b8a75178 was 1745:2402b8a75178, checked in by Stefan Schwarzer <sschwarzer@…>, 4 months ago
Unify formatting of XXX-style comments Use consistent format `# XXX: ...` for all FIXME, TODO and XXX comments.
File size: 39.7 KB
Line 
1# Copyright (C) 2002-2018, Stefan Schwarzer <sschwarzer@sschwarzer.net>
2# and ftputil contributors (see `doc/contributors.txt`)
3# See the file LICENSE for licensing terms.
4
5"""
6`FTPHost` is the central class of the `ftputil` library.
7
8See `__init__.py` for an example.
9"""
10
11import ftplib
12import stat
13import sys
14import time
15
16import ftputil.error
17import ftputil.file
18import ftputil.file_transfer
19import ftputil.path
20import ftputil.stat
21import ftputil.tool
22
23
24__all__ = ["FTPHost"]
25
26
27# The "protected" attributes PyLint talks about aren't intended for
28# clients of the library. `FTPHost` objects need to use some of these
29# library-internal attributes though.
30# pylint: disable=protected-access
31
32
33#####################################################################
34# `FTPHost` class with several methods similar to those of `os`
35
36class FTPHost:
37    """FTP host class."""
38
39    # Implementation notes:
40    #
41    # Upon every request of a file (`FTPFile` object) a new FTP
42    # session is created (or reused from a cache), leading to a child
43    # session of the `FTPHost` object from which the file is requested.
44    #
45    # This is needed because opening an `FTPFile` will make the
46    # local session object wait for the completion of the transfer.
47    # In fact, code like this would block indefinitely, if the `RETR`
48    # request would be made on the `_session` of the object host:
49    #
50    #   host = FTPHost(ftp_server, user, password)
51    #   f = host.open("index.html")
52    #   host.getcwd()   # would block!
53    #
54    # On the other hand, the initially constructed host object will
55    # store references to already established `FTPFile` objects and
56    # reuse an associated connection if its associated `FTPFile`
57    # has been closed.
58
59    def __init__(self, *args, **kwargs):
60        """Abstract initialization of `FTPHost` object."""
61        # Store arguments for later operations.
62        self._args = args
63        self._kwargs = kwargs
64        # XXX: Maybe put the following in a `reset` method.
65        # The time shift setting shouldn't be reset though.
66        # Make a session according to these arguments.
67        self._session = self._make_session()
68        # Simulate `os.path`.
69        self.path = ftputil.path._Path(self)
70        # lstat, stat, listdir services.
71        self._stat = ftputil.stat._Stat(self)
72        self.stat_cache = self._stat._lstat_cache
73        self.stat_cache.enable()
74        with ftputil.error.ftplib_error_to_ftp_os_error:
75            self._cached_current_dir = \
76              self.path.normpath(ftputil.tool.as_unicode(self._session.pwd()))
77        # Associated `FTPHost` objects for data transfer.
78        self._children = []
79        # This is only set to something else than `None` if this
80        # instance represents an `FTPFile`.
81        self._file = None
82        # Now opened.
83        self.closed = False
84        # Set curdir, pardir etc. for the remote host. RFC 959 states
85        # that this is, strictly speaking, dependent on the server OS
86        # but it seems to work at least with Unix and Windows servers.
87        self.curdir, self.pardir, self.sep = ".", "..", "/"
88        # Set default time shift (used in `upload_if_newer` and
89        # `download_if_newer`).
90        self.set_time_shift(0.0)
91        # Don't use `LIST -a` option by default. If the server doesn't
92        # understand the `-a` option and interprets it as a path, the
93        # results can be surprising. See ticket #110.
94        self.use_list_a_option = False
95
96    def keep_alive(self):
97        """
98        Try to keep the connection alive in order to avoid server timeouts.
99
100        Note that this won't help if the connection has already timed
101        out! In this case, `keep_alive` will raise an `TemporaryError`.
102        (Actually, if you get a server timeout, the error - for a specific
103        connection - will be permanent.)
104        """
105        # Warning: Don't call this method on `FTPHost` instances which
106        # represent file transfers. This may fail in confusing ways.
107        with ftputil.error.ftplib_error_to_ftp_os_error:
108            # Ignore return value.
109            self._session.pwd()
110
111    #
112    # Dealing with child sessions and file-like objects
113    # (rather low-level)
114    #
115    def _make_session(self):
116        """
117        Return a new session object according to the current state of
118        this `FTPHost` instance.
119        """
120        # Don't modify original attributes below.
121        args = self._args[:]
122        kwargs = self._kwargs.copy()
123        # If a session factory has been given on the instantiation of
124        # this `FTPHost` object, use the same factory for this
125        # `FTPHost` object's child sessions.
126        factory = kwargs.pop("session_factory", ftplib.FTP)
127        with ftputil.error.ftplib_error_to_ftp_os_error:
128            session = factory(*args, **kwargs)
129        return session
130
131    def _copy(self):
132        """Return a copy of this `FTPHost` object."""
133        # The copy includes a new session factory return value (aka
134        # session) but doesn't copy the state of `self.getcwd()`.
135        return self.__class__(*self._args, **self._kwargs)
136
137    def _available_child(self):
138        """
139        Return an available (i. e. one whose `_file` object is closed
140        and doesn't have a timed-out server connection) child
141        (`FTPHost` object) from the pool of children or `None` if
142        there aren't any.
143        """
144        # TODO: Currently timed-out child sessions aren't removed and
145        # may collect over time. In very busy or long running
146        # processes, this might slow down an application because the
147        # same stale child sessions have to be processed again and
148        # again.
149        for host in self._children:
150            # Test for timeouts only after testing for a closed file:
151            # - If a file isn't closed, save time; don't bother to access
152            #   the remote server.
153            # - If a file transfer on the child is in progress, requesting
154            #   the directory is an invalid operation because of the way
155            #   the FTP state machine works (see RFC 959).
156            if host._file.closed:
157                try:
158                    host._session.pwd()
159                # Under high load, a 226 status response from a
160                # previous download may arrive too late, so that it's
161                # "seen" in the `pwd` call. For now, skip the
162                # potential child session; it will be considered again
163                # when `_available_child` is called the next time.
164                except ftplib.error_reply:
165                    continue
166                # Timed-out sessions raise `error_temp`.
167                except ftplib.error_temp:
168                    continue
169                # The server may have closed the connection which may
170                # cause `host._session.getline` to raise an `EOFError`
171                # (see ticket #114).
172                except EOFError:
173                    continue
174                # Under high load, there may be a socket read timeout
175                # during the last FTP file `close` (see ticket #112).
176                # Note that a socket timeout is quite different from
177                # an FTP session timeout.
178                except OSError:
179                    continue
180                else:
181                    # Everything's ok; use this `FTPHost` instance.
182                    return host
183        # Be explicit.
184        return None
185
186    def open(self, path, mode="r", buffering=None, encoding=None, errors=None,
187             newline=None, rest=None):
188        """
189        Return an open file(-like) object which is associated with
190        this `FTPHost` object.
191
192        The arguments `path`, `mode`, `buffering`, `encoding`,
193        `errors` and `newlines` have the same meaning as for `open`.
194        If `rest` is given as an integer,
195
196        - reading will start at the byte (zero-based) `rest`
197        - writing will overwrite the remote file from byte `rest`
198
199        This method tries to reuse a child but will generate a new one
200        if none is available.
201        """
202        # Support the same arguments as `open`.
203        # pylint: disable=too-many-arguments
204        path = ftputil.tool.as_unicode(path)
205        host = self._available_child()
206        if host is None:
207            host = self._copy()
208            self._children.append(host)
209            host._file = ftputil.file.FTPFile(host)
210        basedir = self.getcwd()
211        # Prepare for changing the directory (see whitespace workaround
212        # in method `_dir`).
213        if host.path.isabs(path):
214            effective_path = path
215        else:
216            effective_path = host.path.join(basedir, path)
217        effective_dir, effective_file = host.path.split(effective_path)
218        try:
219            # This will fail if the directory isn't accessible at all.
220            host.chdir(effective_dir)
221        except ftputil.error.PermanentError:
222            # Similarly to a failed `file` in a local file system,
223            # raise an `IOError`, not an `OSError`.
224            raise ftputil.error.FTPIOError("remote directory '{}' doesn't "
225                    "exist or has insufficient access rights".
226                    format(effective_dir))
227        host._file._open(effective_file, mode=mode, buffering=buffering,
228                         encoding=encoding, errors=errors, newline=newline,
229                         rest=rest)
230        if "w" in mode:
231            # Invalidate cache entry because size and timestamps will change.
232            self.stat_cache.invalidate(effective_path)
233        return host._file
234
235    def close(self):
236        """Close host connection."""
237        if self.closed:
238            return
239        # Close associated children.
240        for host in self._children:
241            # Children have a `_file` attribute which is an `FTPFile` object.
242            host._file.close()
243            host.close()
244        # Now deal with ourself.
245        try:
246            with ftputil.error.ftplib_error_to_ftp_os_error:
247                self._session.close()
248        finally:
249            # If something went wrong before, the host/session is
250            # probably defunct and subsequent calls to `close` won't
251            # help either, so consider the host/session closed for
252            # practical purposes.
253            self.stat_cache.clear()
254            self._children = []
255            self.closed = True
256
257    #
258    # Setting a custom directory parser
259    #
260    def set_parser(self, parser):
261        """
262        Set the parser for extracting stat results from directory
263        listings.
264
265        The parser interface is described in the documentation, but
266        here are the most important things:
267
268        - A parser should derive from `ftputil.stat.Parser`.
269
270        - The parser has to implement two methods, `parse_line` and
271          `ignores_line`. For the latter, there's a probably useful
272          default in the class `ftputil.stat.Parser`.
273
274        - `parse_line` should try to parse a line of a directory
275          listing and return a `ftputil.stat.StatResult` instance. If
276          parsing isn't possible, raise `ftputil.error.ParserError`
277          with a useful error message.
278
279        - `ignores_line` should return a true value if the line isn't
280          assumed to contain stat information.
281        """
282        # The cache contents, if any, probably aren't useful.
283        self.stat_cache.clear()
284        # Set the parser explicitly, don't allow "smart" switching anymore.
285        self._stat._parser = parser
286        self._stat._allow_parser_switching = False
287
288    #
289    # Time shift adjustment between client (i. e. us) and server
290    #
291    def set_time_shift(self, time_shift):
292        """
293        Set the time shift value (i. e. the time difference between
294        client and server) for this `FTPHost` object. By (my)
295        definition, the time shift value is positive if the local
296        time of the server is greater than the local time of the
297        client (for the same physical time), i. e.
298
299            time_shift =def= t_server - t_client
300        <=> t_server = t_client + time_shift
301        <=> t_client = t_server - time_shift
302
303        The time shift is measured in seconds.
304        """
305        # Implicitly set via `set_time_shift` call in constructor
306        # pylint: disable=attribute-defined-outside-init
307        self._time_shift = time_shift
308
309    def time_shift(self):
310        """
311        Return the time shift between FTP server and client. See the
312        docstring of `set_time_shift` for more on this value.
313        """
314        return self._time_shift
315
316    @staticmethod
317    def __rounded_time_shift(time_shift):
318        """
319        Return the given time shift in seconds, but rounded to
320        15-minute units. The argument is also assumed to be given in
321        seconds.
322        """
323        minute = 60.0
324        # Avoid division by zero below.
325        if time_shift == 0:
326            return 0.0
327        # Use a positive value for rounding.
328        absolute_time_shift = abs(time_shift)
329        signum = time_shift / absolute_time_shift
330        # Round absolute time shift to 15-minute units.
331        absolute_rounded_time_shift = (
332          int( (absolute_time_shift + (7.5*minute)) / (15.0*minute) ) *
333          (15.0*minute))
334        # Return with correct sign.
335        return signum * absolute_rounded_time_shift
336
337    def __assert_valid_time_shift(self, time_shift):
338        """
339        Perform sanity checks on the time shift value (given in
340        seconds). If the value is invalid, raise a `TimeShiftError`,
341        else simply return `None`.
342        """
343        minute = 60.0  # seconds
344        hour = 60.0 * minute
345        absolute_rounded_time_shift = \
346          abs(self.__rounded_time_shift(time_shift))
347        # Test 1: Fail if the absolute time shift is greater than
348        #         a full day (24 hours).
349        if absolute_rounded_time_shift > 24 * hour:
350            raise ftputil.error.TimeShiftError(
351                  "time shift abs({0:.2f} s) > 1 day".format(time_shift))
352        # Test 2: Fail if the deviation between given time shift and
353        #         15-minute units is greater than a certain limit.
354        maximum_deviation = 5 * minute
355        if abs(time_shift - self.__rounded_time_shift(time_shift)) > \
356           maximum_deviation:
357            raise ftputil.error.TimeShiftError(
358                    "time shift ({0:.2f} s) deviates more than {1:d} s "
359                    "from 15-minute units".format(
360                      time_shift, int(maximum_deviation)))
361
362    def synchronize_times(self):
363        """
364        Synchronize the local times of FTP client and server. This
365        is necessary to let `upload_if_newer` and `download_if_newer`
366        work correctly. If `synchronize_times` isn't applicable
367        (see below), the time shift can still be set explicitly with
368        `set_time_shift`.
369
370        This implementation of `synchronize_times` requires _all_ of
371        the following:
372
373        - The connection between server and client is established.
374        - The client has write access to the directory that is
375          current when `synchronize_times` is called.
376
377        The common usage pattern of `synchronize_times` is to call it
378        directly after the connection is established. (As can be
379        concluded from the points above, this requires write access
380        to the login directory.)
381
382        If `synchronize_times` fails, it raises a `TimeShiftError`.
383        """
384        helper_file_name = "_ftputil_sync_"
385        # Open a dummy file for writing in the current directory
386        # on the FTP host, then close it.
387        try:
388            # May raise `FTPIOError` if directory isn't writable.
389            file_ = self.open(helper_file_name, "w")
390            file_.close()
391        except ftputil.error.FTPIOError:
392            raise ftputil.error.TimeShiftError(
393                    """couldn't write helper file in directory '{}'""".
394                    format(self.getcwd()))
395        # If everything worked up to here it should be possible to stat
396        # and then remove the just-written file.
397        try:
398            server_time = self.path.getmtime(helper_file_name)
399            self.unlink(helper_file_name)
400        except ftputil.error.FTPOSError:
401            # If we got a `TimeShiftError` exception above, we should't
402            # come here: if we did not get a `TimeShiftError` above,
403            # deletion should be possible. The only reason for an exception
404            # I can think of here is a race condition by removing write
405            # permission from the directory or helper file after it has been
406            # written to.
407            raise ftputil.error.TimeShiftError(
408                    "could write helper file but not unlink it")
409        # Calculate the difference between server and client.
410        now = time.time()
411        time_shift = server_time - now
412        # As the time shift for this host instance isn't set yet, the
413        # directory parser will calculate times one year in the past if
414        # the time zone of the server is east from ours. Thus the time
415        # shift will be off by a year as well (see ticket #55).
416        if time_shift < -360 * 24 * 60 * 60:
417            # Re-add one year and re-calculate the time shift. We don't
418            # know how many days made up that year (it might have been
419            # a leap year), so go the route via `time.localtime` and
420            # `time.mktime`.
421            server_time_struct = time.localtime(server_time)
422            server_time_struct = (server_time_struct.tm_year+1,) + \
423                                 server_time_struct[1:]
424            server_time = time.mktime(server_time_struct)
425            time_shift = server_time - now
426        # Do some sanity checks.
427        self.__assert_valid_time_shift(time_shift)
428        # If tests passed, store the time difference as time shift value.
429        self.set_time_shift(self.__rounded_time_shift(time_shift))
430
431    #
432    # Operations based on file-like objects (rather high-level),
433    # like upload and download
434    #
435    # This code doesn't complain if the chunk size is passed as a
436    # positional argument but emits a deprecation warning if `length`
437    # is used as a keyword argument.
438    @staticmethod
439    def copyfileobj(source, target,
440                    max_chunk_size=ftputil.file_transfer.MAX_COPY_CHUNK_SIZE,
441                    callback=None):
442        """
443        Copy data from file-like object `source` to file-like object
444        `target`.
445        """
446        ftputil.file_transfer.copyfileobj(source, target, max_chunk_size,
447                                          callback)
448
449    def _upload_files(self, source_path, target_path):
450        """
451        Return a `LocalFile` and `RemoteFile` as source and target,
452        respectively.
453
454        The strings `source_path` and `target_path` are the (absolute
455        or relative) paths of the local and the remote file, respectively.
456        """
457        source_file = ftputil.file_transfer.LocalFile(source_path, "rb")
458        # Passing `self` (the `FTPHost` instance) here is correct.
459        target_file = ftputil.file_transfer.RemoteFile(self, target_path, "wb")
460        return source_file, target_file
461
462    def upload(self, source, target, callback=None):
463        """
464        Upload a file from the local source (name) to the remote
465        target (name).
466
467        If a callable `callback` is given, it's called after every
468        chunk of transferred data. The chunk size is a constant
469        defined in `file_transfer`. The callback will be called with a
470        single argument, the data chunk that was transferred before
471        the callback was called.
472        """
473        target = ftputil.tool.as_unicode(target)
474        source_file, target_file = self._upload_files(source, target)
475        ftputil.file_transfer.copy_file(source_file, target_file,
476                                        conditional=False, callback=callback)
477
478    def upload_if_newer(self, source, target, callback=None):
479        """
480        Upload a file only if it's newer than the target on the
481        remote host or if the target file does not exist. See the
482        method `upload` for the meaning of the parameters.
483
484        If an upload was necessary, return `True`, else return `False`.
485
486        If a callable `callback` is given, it's called after every
487        chunk of transferred data. The chunk size is a constant
488        defined in `file_transfer`. The callback will be called with a
489        single argument, the data chunk that was transferred before
490        the callback was called.
491        """
492        target = ftputil.tool.as_unicode(target)
493        source_file, target_file = self._upload_files(source, target)
494        return ftputil.file_transfer.copy_file(source_file, target_file,
495                                               conditional=True,
496                                               callback=callback)
497
498    def _download_files(self, source_path, target_path):
499        """
500        Return a `RemoteFile` and `LocalFile` as source and target,
501        respectively.
502
503        The strings `source_path` and `target_path` are the (absolute
504        or relative) paths of the remote and the local file, respectively.
505        """
506        source_file = ftputil.file_transfer.RemoteFile(self, source_path, "rb")
507        target_file = ftputil.file_transfer.LocalFile(target_path, "wb")
508        return source_file, target_file
509
510    def download(self, source, target, callback=None):
511        """
512        Download a file from the remote source (name) to the local
513        target (name).
514
515        If a callable `callback` is given, it's called after every
516        chunk of transferred data. The chunk size is a constant
517        defined in `file_transfer`. The callback will be called with a
518        single argument, the data chunk that was transferred before
519        the callback was called.
520        """
521        source = ftputil.tool.as_unicode(source)
522        source_file, target_file = self._download_files(source, target)
523        ftputil.file_transfer.copy_file(source_file, target_file,
524                                        conditional=False, callback=callback)
525
526    def download_if_newer(self, source, target, callback=None):
527        """
528        Download a file only if it's newer than the target on the
529        local host or if the target file does not exist. See the
530        method `download` for the meaning of the parameters.
531
532        If a download was necessary, return `True`, else return
533        `False`.
534
535        If a callable `callback` is given, it's called after every
536        chunk of transferred data. The chunk size is a constant
537        defined in `file_transfer`. The callback will be called with a
538        single argument, the data chunk that was transferred before
539        the callback was called.
540        """
541        source = ftputil.tool.as_unicode(source)
542        source_file, target_file = self._download_files(source, target)
543        return ftputil.file_transfer.copy_file(source_file, target_file,
544                                               conditional=True,
545                                               callback=callback)
546
547    #
548    # Helper methods to descend into a directory before executing a command
549    #
550    def _check_inaccessible_login_directory(self):
551        """
552        Raise an `InaccessibleLoginDirError` exception if we can't
553        change to the login directory. This test is only reliable if
554        the current directory is the login directory.
555        """
556        presumable_login_dir = self.getcwd()
557        # Bail out with an internal error rather than modify the
558        # current directory without hope of restoration.
559        try:
560            self.chdir(presumable_login_dir)
561        except ftputil.error.PermanentError:
562            raise ftputil.error.InaccessibleLoginDirError(
563                    "directory '{}' is not accessible".
564                    format(presumable_login_dir))
565
566    def _robust_ftp_command(self, command, path, descend_deeply=False):
567        """
568        Run an FTP command on a path. The return value of the method
569        is the return value of the command.
570
571        If `descend_deeply` is true (the default is false), descend
572        deeply, i. e. change the directory to the end of the path.
573        """
574        # If we can't change to the yet-current directory, the code
575        # below won't work (see below), so in this case rather raise
576        # an exception than giving wrong results.
577        self._check_inaccessible_login_directory()
578        # Some FTP servers don't behave as expected if the directory
579        # portion of the path contains whitespace; some even yield
580        # strange results if the command isn't executed in the
581        # current directory. Therefore, change to the directory
582        # which contains the item to run the command on and invoke
583        # the command just there.
584        #
585        # Remember old working directory.
586        old_dir = self.getcwd()
587        try:
588            if descend_deeply:
589                # Invoke the command in (not: on) the deepest directory.
590                self.chdir(path)
591                # Workaround for some servers that give recursive
592                # listings when called with a dot as path; see issue #33,
593                # http://ftputil.sschwarzer.net/trac/ticket/33
594                return command(self, "")
595            else:
596                # Invoke the command in the "next to last" directory.
597                head, tail = self.path.split(path)
598                self.chdir(head)
599                return command(self, tail)
600        finally:
601            self.chdir(old_dir)
602
603    #
604    # Miscellaneous utility methods resembling functions in `os`
605    #
606    def getcwd(self):
607        """Return the current path name."""
608        return self._cached_current_dir
609
610    def chdir(self, path):
611        """Change the directory on the host."""
612        path = ftputil.tool.as_unicode(path)
613        with ftputil.error.ftplib_error_to_ftp_os_error:
614            self._session.cwd(path)
615        # The path given as the argument is relative to the old current
616        # directory, therefore join them.
617        self._cached_current_dir = \
618          self.path.normpath(self.path.join(self._cached_current_dir, path))
619
620    # Ignore unused argument `mode`
621    # pylint: disable=unused-argument
622    def mkdir(self, path, mode=None):
623        """
624        Make the directory path on the remote host. The argument
625        `mode` is ignored and only "supported" for similarity with
626        `os.mkdir`.
627        """
628        path = ftputil.tool.as_unicode(path)
629        def command(self, path):
630            """Callback function."""
631            with ftputil.error.ftplib_error_to_ftp_os_error:
632                self._session.mkd(path)
633        self._robust_ftp_command(command, path)
634
635    # TODO: The virtual directory support doesn't have unit tests yet
636    # because the mocking most likely would be quite complicated. The
637    # tests should be added when mainly the `mock` library is used
638    # instead of the mock code in `test.mock_ftplib`.
639    #
640    # Ignore unused argument `mode`
641    # pylint: disable=unused-argument
642    def makedirs(self, path, mode=None):
643        """
644        Make the directory `path`, but also make not yet existing
645        intermediate directories, like `os.makedirs`. The value of
646        `mode` is only accepted for compatibility with `os.makedirs`
647        but otherwise ignored.
648        """
649        path = ftputil.tool.as_unicode(path)
650        path = self.path.abspath(path)
651        directories = path.split(self.sep)
652        old_dir = self.getcwd()
653        try:
654            # Try to build the directory chain from the "uppermost" to
655            # the "lowermost" directory.
656            for index in range(1, len(directories)):
657                # Re-insert the separator which got lost by using
658                # `path.split`.
659                next_directory = (self.sep +
660                                  self.path.join(*directories[:index+1]))
661                # If we have "virtual directories" (see #86), just
662                # listing the parent directory won't tell us if a
663                # directory actually exists. So try to change into the
664                # directory.
665                try:
666                    self.chdir(next_directory)
667                except ftputil.error.PermanentError:
668                    try:
669                        self.mkdir(next_directory)
670                    except ftputil.error.PermanentError:
671                        # Find out the cause of the error. Re-raise
672                        # the exception only if the directory didn't
673                        # exist already, else something went _really_
674                        # wrong, e. g. there's a regular file with the
675                        # name of the directory.
676                        if not self.path.isdir(next_directory):
677                            raise
678        finally:
679            self.chdir(old_dir)
680
681    def rmdir(self, path):
682        """
683        Remove the _empty_ directory `path` on the remote host.
684
685        Compatibility note:
686
687        Previous versions of ftputil could possibly delete non-
688        empty directories as well, - if the server allowed it. This
689        is no longer supported.
690        """
691        path = ftputil.tool.as_unicode(path)
692        path = self.path.abspath(path)
693        if self.listdir(path):
694            raise ftputil.error.PermanentError("directory '{}' not empty".
695                                               format(path))
696        # XXX: How does `rmd` work with links?
697        def command(self, path):
698            """Callback function."""
699            with ftputil.error.ftplib_error_to_ftp_os_error:
700                self._session.rmd(path)
701        self._robust_ftp_command(command, path)
702        self.stat_cache.invalidate(path)
703
704    def remove(self, path):
705        """
706        Remove the file or link given by `path`.
707
708        Raise a `PermanentError` if the path doesn't exist, but maybe
709        raise other exceptions depending on the state of the server
710        (e. g. timeout).
711        """
712        path = ftputil.tool.as_unicode(path)
713        path = self.path.abspath(path)
714        # Though `isfile` includes also links to files, `islink`
715        # is needed to include links to directories.
716        if self.path.isfile(path) or self.path.islink(path) or \
717           not self.path.exists(path):
718            # If the path doesn't exist, let the removal command trigger
719            # an exception with a more appropriate error message.
720            def command(self, path):
721                """Callback function."""
722                with ftputil.error.ftplib_error_to_ftp_os_error:
723                    self._session.delete(path)
724            self._robust_ftp_command(command, path)
725        else:
726            raise ftputil.error.PermanentError(
727                    "remove/unlink can only delete files and links, "
728                    "not directories")
729        self.stat_cache.invalidate(path)
730
731    unlink = remove
732
733    def rmtree(self, path, ignore_errors=False, onerror=None):
734        """
735        Remove the given remote, possibly non-empty, directory tree.
736        The interface of this method is rather complex, in favor of
737        compatibility with `shutil.rmtree`.
738
739        If `ignore_errors` is set to a true value, errors are ignored.
740        If `ignore_errors` is a false value _and_ `onerror` isn't set,
741        all exceptions occurring during the tree iteration and
742        processing are raised. These exceptions are all of type
743        `PermanentError`.
744
745        To distinguish between error situations, pass in a callable
746        for `onerror`. This callable must accept three arguments:
747        `func`, `path` and `exc_info`. `func` is a bound method
748        object, _for example_ `your_host_object.listdir`. `path` is
749        the path that was the recent argument of the respective method
750        (`listdir`, `remove`, `rmdir`). `exc_info` is the exception
751        info as it's got from `sys.exc_info`.
752
753        Implementation note: The code is copied from `shutil.rmtree`
754        in Python 2.4 and adapted to ftputil.
755        """
756        path = ftputil.tool.as_unicode(path)
757        # The following code is an adapted version of Python 2.4's
758        # `shutil.rmtree` function.
759        if ignore_errors:
760            def new_onerror(*args):
761                """Do nothing."""
762                # pylint: disable=unused-argument
763                pass
764        elif onerror is None:
765            def new_onerror(*args):
766                """Re-raise exception."""
767                # pylint: disable=misplaced-bare-raise, unused-argument
768                raise
769        else:
770            new_onerror = onerror
771        names = []
772        try:
773            names = self.listdir(path)
774        except ftputil.error.PermanentError:
775            new_onerror(self.listdir, path, sys.exc_info())
776        for name in names:
777            full_name = self.path.join(path, name)
778            try:
779                mode = self.lstat(full_name).st_mode
780            except ftputil.error.PermanentError:
781                mode = 0
782            if stat.S_ISDIR(mode):
783                self.rmtree(full_name, ignore_errors, new_onerror)
784            else:
785                try:
786                    self.remove(full_name)
787                except ftputil.error.PermanentError:
788                    new_onerror(self.remove, full_name, sys.exc_info())
789        try:
790            self.rmdir(path)
791        except ftputil.error.FTPOSError:
792            new_onerror(self.rmdir, path, sys.exc_info())
793
794    def rename(self, source, target):
795        """Rename the source on the FTP host to target."""
796        source = ftputil.tool.as_unicode(source)
797        target = ftputil.tool.as_unicode(target)
798        # The following code is in spirit similar to the code in the
799        # method `_robust_ftp_command`, though we do _not_ do
800        # _everything_ imaginable.
801        self._check_inaccessible_login_directory()
802        source_head, source_tail = self.path.split(source)
803        target_head, target_tail = self.path.split(target)
804        paths_contain_whitespace = (" " in source_head) or (" " in target_head)
805        if paths_contain_whitespace and source_head == target_head:
806            # Both items are in the same directory.
807            old_dir = self.getcwd()
808            try:
809                self.chdir(source_head)
810                with ftputil.error.ftplib_error_to_ftp_os_error:
811                    self._session.rename(source_tail, target_tail)
812            finally:
813                self.chdir(old_dir)
814        else:
815            # Use straightforward command.
816            with ftputil.error.ftplib_error_to_ftp_os_error:
817                self._session.rename(source, target)
818
819    # XXX: One could argue to put this method into the `_Stat` class,
820    # but I refrained from that because then `_Stat` would have to
821    # know about `FTPHost`'s `_session` attribute and in turn about
822    # `_session`'s `dir` method.
823    def _dir(self, path):
824        """Return a directory listing as made by FTP's `LIST` command."""
825        # Don't use `self.path.isdir` in this method because that
826        # would cause a call of `(l)stat` and thus a call to `_dir`,
827        # so we would end up with an infinite recursion.
828        def _FTPHost_dir_command(self, path):
829            """Callback function."""
830            lines = []
831            def callback(line):
832                """Callback function."""
833                lines.append(ftputil.tool.as_unicode(line))
834            with ftputil.error.ftplib_error_to_ftp_os_error:
835                if self.use_list_a_option:
836                    self._session.dir("-a", path, callback)
837                else:
838                    self._session.dir(path, callback)
839            return lines
840        lines = self._robust_ftp_command(_FTPHost_dir_command, path,
841                                         descend_deeply=True)
842        return lines
843
844    # The `listdir`, `lstat` and `stat` methods don't use
845    # `_robust_ftp_command` because they implicitly already use
846    # `_dir` which actually uses `_robust_ftp_command`.
847    def listdir(self, path):
848        """
849        Return a list of directories, files etc. in the directory
850        named `path`.
851
852        If the directory listing from the server can't be parsed with
853        any of the available parsers raise a `ParserError`.
854        """
855        original_path = path
856        path = ftputil.tool.as_unicode(path)
857        items = self._stat._listdir(path)
858        return [ftputil.tool.same_string_type_as(original_path, item)
859                for item in items]
860
861    def lstat(self, path, _exception_for_missing_path=True):
862        """
863        Return an object similar to that returned by `os.lstat`.
864
865        If the directory listing from the server can't be parsed with
866        any of the available parsers, raise a `ParserError`. If the
867        directory _can_ be parsed and the `path` is _not_ found, raise
868        a `PermanentError`.
869
870        (`_exception_for_missing_path` is an implementation aid and
871        _not_ intended for use by ftputil clients.)
872        """
873        path = ftputil.tool.as_unicode(path)
874        return self._stat._lstat(path, _exception_for_missing_path)
875
876    def stat(self, path, _exception_for_missing_path=True):
877        """
878        Return info from a "stat" call on `path`.
879
880        If the directory containing `path` can't be parsed, raise a
881        `ParserError`. If the directory containing `path` can be
882        parsed but the `path` can't be found, raise a
883        `PermanentError`. Also raise a `PermanentError` if there's an
884        endless (cyclic) chain of symbolic links "behind" the `path`.
885
886        (`_exception_for_missing_path` is an implementation aid and
887        _not_ intended for use by ftputil clients.)
888        """
889        path = ftputil.tool.as_unicode(path)
890        return self._stat._stat(path, _exception_for_missing_path)
891
892    def walk(self, top, topdown=True, onerror=None, followlinks=False):
893        """
894        Iterate over directory tree and return a tuple (dirpath,
895        dirnames, filenames) on each iteration, like the `os.walk`
896        function (see https://docs.python.org/library/os.html#os.walk ).
897        """
898        top = ftputil.tool.as_unicode(top)
899        # The following code is copied from `os.walk` in Python 2.4
900        # and adapted to ftputil.
901        try:
902            names = self.listdir(top)
903        except ftputil.error.FTPOSError as err:
904            if onerror is not None:
905                onerror(err)
906            return
907        dirs, nondirs = [], []
908        for name in names:
909            if self.path.isdir(self.path.join(top, name)):
910                dirs.append(name)
911            else:
912                nondirs.append(name)
913        if topdown:
914            yield top, dirs, nondirs
915        for name in dirs:
916            path = self.path.join(top, name)
917            if followlinks or not self.path.islink(path):
918                for item in self.walk(path, topdown, onerror, followlinks):
919                    yield item
920        if not topdown:
921            yield top, dirs, nondirs
922
923    def chmod(self, path, mode):
924        """
925        Change the mode of a remote `path` (a string) to the integer
926        `mode`. This integer uses the same bits as the mode value
927        returned by the `stat` and `lstat` commands.
928
929        If something goes wrong, raise a `TemporaryError` or a
930        `PermanentError`, according to the status code returned by
931        the server. In particular, a non-existent path usually
932        causes a `PermanentError`.
933        """
934        path = ftputil.tool.as_unicode(path)
935        path = self.path.abspath(path)
936        def command(self, path):
937            """Callback function."""
938            with ftputil.error.ftplib_error_to_ftp_os_error:
939                self._session.voidcmd("SITE CHMOD 0{0:o} {1}".
940                                      format(mode, path))
941        self._robust_ftp_command(command, path)
942        self.stat_cache.invalidate(path)
943
944    def __getstate__(self):
945        raise TypeError("cannot serialize FTPHost object")
946
947    #
948    # Context manager methods
949    #
950    def __enter__(self):
951        # Return `self`, so it can be accessed as the variable
952        # component of the `with` statement.
953        return self
954
955    def __exit__(self, exc_type, exc_val, exc_tb):
956        # We don't need the `exc_*` arguments here.
957        # pylint: disable=unused-argument
958        self.close()
959        # Be explicit.
960        return False
Note: See TracBrowser for help on using the repository browser.