Ticket #145: host.py

File host.py, 42.0 KB (added by schwa, 7 months ago)

host.py with fix for ticket #145

Line 
1# Copyright (C) 2002-2021, Stefan Schwarzer <sschwarzer@sschwarzer.net>
2# and ftputil contributors (see `doc/contributors.txt`)
3# See the file LICENSE for licensing terms.
4
5"""
6`FTPHost` is the central class of the `ftputil` library.
7
8See `__init__.py` for an example.
9"""
10
11import datetime
12import errno
13import ftplib
14import stat
15import sys
16import time
17
18import ftputil.error
19import ftputil.file
20import ftputil.file_transfer
21import ftputil.path
22import ftputil.path_encoding
23import ftputil.session
24import ftputil.stat
25import ftputil.tool
26
27
28__all__ = ["FTPHost"]
29
30
31# The "protected" attributes PyLint talks about aren't intended for clients of
32# the library. `FTPHost` objects need to use some of these library-internal
33# attributes though.
34# pylint: disable=protected-access
35
36
37# For Python versions 3.8 and below, ftputil has implicitly defaulted to
38# latin-1 encoding. Prefer that behavior for Python 3.9 and up as well instead
39# of using the encoding that is the default for `ftplib.FTP` in the Python
40# version.
41if ftputil.path_encoding.RUNNING_UNDER_PY39_AND_UP:
42
43    class default_session_factory(ftplib.FTP):
44        def __init__(self, *args, **kwargs):
45            # Python 3.9 defines `encoding` as a keyword-only argument, so test
46            # only for the `encoding` argument in `kwargs`.
47            #
48            # Only use the ftputil default encoding if the caller didn't pass
49            # an encoding.
50            if "encoding" not in kwargs:
51                kwargs["encoding"] = ftputil.path_encoding.DEFAULT_ENCODING
52            super().__init__(*args, **kwargs)
53
54
55else:
56    default_session_factory = ftplib.FTP
57
58
59#####################################################################
60# `FTPHost` class with several methods similar to those of `os`
61
62
63class FTPHost:
64    """
65    FTP host class.
66    """
67
68    # Implementation notes:
69    #
70    # Upon every request of a file (`FTPFile` object) a new FTP session is
71    # created (or reused from a cache), leading to a child session of the
72    # `FTPHost` object from which the file is requested.
73    #
74    # This is needed because opening an `FTPFile` will make the local session
75    # object wait for the completion of the transfer. In fact, code like this
76    # would block indefinitely, if the `RETR` request would be made on the
77    # `_session` of the object host:
78    #
79    #   host = FTPHost(ftp_server, user, password)
80    #   f = host.open("index.html")
81    #   host.getcwd()   # would block!
82    #
83    # On the other hand, the initially constructed host object will store
84    # references to already established `FTPFile` objects and reuse an
85    # associated connection if its associated `FTPFile` has been closed.
86
87    def __init__(self, *args, **kwargs):
88        """
89        Abstract initialization of `FTPHost` object.
90        """
91        # Store arguments for later operations.
92        self._args = args
93        self._kwargs = kwargs
94        # XXX: Maybe put the following in a `reset` method.
95        # The time shift setting shouldn't be reset though. Make a session
96        # according to these arguments.
97        self._session = self._make_session()
98        # Simulate `os.path`.
99        self.path = ftputil.path._Path(self)
100        # lstat, stat, listdir services.
101        self._stat = ftputil.stat._Stat(self)
102        self.stat_cache = self._stat._lstat_cache
103        self.stat_cache.enable()
104        with ftputil.error.ftplib_error_to_ftp_os_error:
105            current_dir = self._session.pwd()
106            self._cached_current_dir = self.path.normpath(
107                ftputil.tool.as_str_path(current_dir, encoding=self._encoding)
108            )
109        # Associated `FTPHost` objects for data transfer.
110        self._children = []
111        # This is only set to something else than `None` if this instance
112        # represents an `FTPFile`.
113        self._file = None
114        # Now opened.
115        self.closed = False
116        # Set curdir, pardir etc. for the remote host. RFC 959 states that this
117        # is, strictly speaking, dependent on the server OS but it seems to
118        # work at least with Unix and Windows servers.
119        self.curdir, self.pardir, self.sep = ".", "..", "/"
120        # Set default time shift (used in `upload_if_newer` and
121        # `download_if_newer`).
122        self._time_shift = 0.0
123        # Don't use `LIST -a` option by default. If the server doesn't
124        # understand the `-a` option and interprets it as a path, the results
125        # can be surprising. See ticket #110.
126        self.use_list_a_option = False
127
128    def keep_alive(self):
129        """
130        Try to keep the connection alive in order to avoid server timeouts.
131
132        Note that this won't help if the connection has already timed out! In
133        this case, `keep_alive` will raise an `TemporaryError`. (Actually, if
134        you get a server timeout, the error - for a specific connection - will
135        be permanent.)
136        """
137        # Warning: Don't call this method on `FTPHost` instances which
138        # represent file transfers. This may fail in confusing ways.
139        with ftputil.error.ftplib_error_to_ftp_os_error:
140            # Ignore return value.
141            self._session.pwd()
142
143    #
144    # Dealing with child sessions and file-like objects (rather low-level)
145    #
146    def _make_session(self):
147        """
148        Return a new session object according to the current state of this
149        `FTPHost` instance.
150        """
151        # Don't modify original attributes below.
152        args = self._args[:]
153        kwargs = self._kwargs.copy()
154        # If a session factory has been given on the instantiation of this
155        # `FTPHost` object, use the same factory for this `FTPHost` object's
156        # child sessions.
157        factory = kwargs.pop("session_factory", default_session_factory)
158        with ftputil.error.ftplib_error_to_ftp_os_error:
159            session = factory(*args, **kwargs)
160            if not hasattr(session, "encoding"):
161                raise ftputil.error.NoEncodingError(
162                    f"session instance {session!r} must have an `encoding` attribute"
163                )
164            self._encoding = session.encoding
165        return session
166
167    def _copy(self):
168        """
169        Return a copy of this `FTPHost` object.
170        """
171        # The copy includes a new session factory return value (aka session)
172        # but doesn't copy the state of `self.getcwd()`.
173        return self.__class__(*self._args, **self._kwargs)
174
175    def _available_child(self):
176        """
177        Return an available (i. e. one whose `_file` object is closed and
178        doesn't have a timed-out server connection) child (`FTPHost` object)
179        from the pool of children or `None` if there aren't any.
180        """
181        # TODO: Currently timed-out child sessions aren't removed and may
182        # collect over time. In very busy or long running processes, this might
183        # slow down an application because the same stale child sessions have
184        # to be processed again and again.
185        for host in self._children:
186            # Test for timeouts only after testing for a closed file:
187            # - If a file isn't closed, save time; don't bother to access the
188            #   remote server.
189            # - If a file transfer on the child is in progress, requesting the
190            #   directory is an invalid operation because of the way the FTP
191            #   state machine works (see RFC 959).
192            if host._file.closed:
193                try:
194                    host._session.pwd()
195                # Under high load, a 226 status response from a previous
196                # download may arrive too late, so that it's "seen" in the
197                # `pwd` call. For now, skip the potential child session; it
198                # will be considered again when `_available_child` is called
199                # the next time.
200                except ftplib.error_reply:
201                    continue
202                # Timed-out sessions raise `error_temp`.
203                except ftplib.error_temp:
204                    continue
205                # The server may have closed the connection which may cause
206                # `host._session.getline` to raise an `EOFError` (see ticket
207                # #114).
208                except EOFError:
209                    continue
210                # Under high load, there may be a socket read timeout during
211                # the last FTP file `close` (see ticket #112). Note that a
212                # socket timeout is quite different from an FTP session
213                # timeout.
214                except OSError:
215                    continue
216                else:
217                    # Everything's ok; use this `FTPHost` instance.
218                    return host
219        # Be explicit.
220        return None
221
222    def open(
223        self,
224        path,
225        mode="r",
226        buffering=None,
227        encoding=None,
228        errors=None,
229        newline=None,
230        *,
231        rest=None,
232    ):
233        """
234        Return an open file(-like) object which is associated with this
235        `FTPHost` object.
236
237        The arguments `path`, `mode`, `buffering`, `encoding`, `errors` and
238        `newlines` have the same meaning as for `open`. If `rest` is given as
239        an integer,
240
241        - reading will start at the byte (zero-based) `rest`
242        - writing will overwrite the remote file from byte `rest`
243
244        This method tries to reuse a child but will generate a new one if none
245        is available.
246        """
247        # Support the same arguments as `open`.
248        # pylint: disable=too-many-arguments
249        path = ftputil.tool.as_str_path(path, encoding=self._encoding)
250        host = self._available_child()
251        if host is None:
252            host = self._copy()
253            self._children.append(host)
254            host._file = ftputil.file.FTPFile(host)
255        basedir = self.getcwd()
256        # Prepare for changing the directory (see whitespace workaround in
257        # method `_dir`).
258        if host.path.isabs(path):
259            effective_path = path
260        else:
261            effective_path = host.path.join(basedir, path)
262        effective_dir, effective_file = host.path.split(effective_path)
263        try:
264            # This will fail if the directory isn't accessible at all.
265            host.chdir(effective_dir)
266        except ftputil.error.PermanentError:
267            # Similarly to a failed `file` in a local file system, raise an
268            # `IOError`, not an `OSError`.
269            raise ftputil.error.FTPIOError(
270                "remote directory '{}' doesn't "
271                "exist or has insufficient access rights".format(effective_dir)
272            )
273        host._file._open(
274            effective_file,
275            mode=mode,
276            buffering=buffering,
277            encoding=encoding,
278            errors=errors,
279            newline=newline,
280            rest=rest,
281        )
282        if "w" in mode:
283            # Invalidate cache entry because size and timestamps will change.
284            self.stat_cache.invalidate(effective_path)
285        return host._file
286
287    def close(self):
288        """
289        Close host connection.
290        """
291        if self.closed:
292            return
293        # Close associated children.
294        for host in self._children:
295            # Children have a `_file` attribute which is an `FTPFile` object.
296            host._file.close()
297            host.close()
298        # Now deal with ourself.
299        try:
300            with ftputil.error.ftplib_error_to_ftp_os_error:
301                self._session.close()
302        finally:
303            # If something went wrong before, the host/session is probably
304            # defunct and subsequent calls to `close` won't help either, so
305            # consider the host/session closed for practical purposes.
306            self.stat_cache.clear()
307            self._children = []
308            self.closed = True
309
310    #
311    # Setting a custom directory parser
312    #
313    def set_parser(self, parser):
314        """
315        Set the parser for extracting stat results from directory listings.
316
317        The parser interface is described in the documentation, but here are
318        the most important things:
319
320        - A parser should derive from `ftputil.stat.Parser`.
321
322        - The parser has to implement two methods, `parse_line` and
323          `ignores_line`. For the latter, there's a probably useful default
324          in the class `ftputil.stat.Parser`.
325
326        - `parse_line` should try to parse a line of a directory listing and
327          return a `ftputil.stat.StatResult` instance. If parsing isn't
328          possible, raise `ftputil.error.ParserError` with a useful error
329          message.
330
331        - `ignores_line` should return a true value if the line isn't assumed
332          to contain stat information.
333        """
334        # The cache contents, if any, probably aren't useful.
335        self.stat_cache.clear()
336        # Set the parser explicitly, don't allow "smart" switching anymore.
337        self._stat._parser = parser
338        self._stat._allow_parser_switching = False
339
340    #
341    # Time shift adjustment between client (i. e. us) and server
342    #
343    @staticmethod
344    def __rounded_time_shift(time_shift):
345        """
346        Return the given time shift in seconds, but rounded to 15-minute units.
347        The argument is also assumed to be given in seconds.
348        """
349        minute = 60.0
350        # Avoid division by zero below.
351        if time_shift == 0:
352            return 0.0
353        # Use a positive value for rounding.
354        absolute_time_shift = abs(time_shift)
355        signum = time_shift / absolute_time_shift
356        # Round absolute time shift to 15-minute units.
357        absolute_rounded_time_shift = int(
358            (absolute_time_shift + (7.5 * minute)) / (15.0 * minute)
359        ) * (15.0 * minute)
360        # Return with correct sign.
361        return signum * absolute_rounded_time_shift
362
363    def __assert_valid_time_shift(self, time_shift):
364        """
365        Perform sanity checks on the time shift value (given in seconds). If
366        the value is invalid, raise a `TimeShiftError`, else simply return
367        `None`.
368        """
369        minute = 60.0  # seconds
370        hour = 60.0 * minute
371        absolute_rounded_time_shift = abs(self.__rounded_time_shift(time_shift))
372        # Test 1: Fail if the absolute time shift is greater than a full day
373        #         (24 hours).
374        if absolute_rounded_time_shift > 24 * hour:
375            raise ftputil.error.TimeShiftError(
376                "time shift abs({0:.2f} s) > 1 day".format(time_shift)
377            )
378        # Test 2: Fail if the deviation between given time shift and 15-minute
379        #         units is greater than a certain limit.
380        maximum_deviation = 5 * minute
381        if abs(time_shift - self.__rounded_time_shift(time_shift)) > maximum_deviation:
382            raise ftputil.error.TimeShiftError(
383                "time shift ({0:.2f} s) deviates more than {1:d} s "
384                "from 15-minute units".format(time_shift, int(maximum_deviation))
385            )
386
387    def set_time_shift(self, time_shift):
388        """
389        Set the time shift value.
390
391        By (my) definition, the time shift value is the difference of
392        the time zone used in server listings and UTC, i. e.
393
394            time_shift =def= t_server - utc
395        <=> t_server = utc + time_shift
396        <=> utc = t_server - time_shift
397
398        The time shift is measured in seconds.
399        """
400        self.__assert_valid_time_shift(time_shift)
401        old_time_shift = self.time_shift()
402        if time_shift != old_time_shift:
403            # If the time shift changed, all entries in the cache will have
404            # wrong times with respect to the updated time shift, therefore
405            # clear the cache.
406            self.stat_cache.clear()
407            self._time_shift = self.__rounded_time_shift(time_shift)
408
409    def time_shift(self):
410        """
411        Return the time shift between FTP server and client. See the
412        docstring of `set_time_shift` for more on this value.
413        """
414        return self._time_shift
415
416    def synchronize_times(self):
417        """
418        Synchronize the local times of FTP client and server. This is necessary
419        to let `upload_if_newer` and `download_if_newer` work correctly. If
420        `synchronize_times` isn't applicable (see below), the time shift can
421        still be set explicitly with `set_time_shift`.
422
423        This implementation of `synchronize_times` requires _all_ of the
424        following:
425
426        - The connection between server and client is established.
427        - The client has write access to the directory that is current when
428          `synchronize_times` is called.
429
430        The common usage pattern of `synchronize_times` is to call it directly
431        after the connection is established. (As can be concluded from the
432        points above, this requires write access to the login directory.)
433
434        If `synchronize_times` fails, it raises a `TimeShiftError`.
435        """
436        helper_file_name = "_ftputil_sync_"
437        # Open a dummy file for writing in the current directory on the FTP
438        # host, then close it.
439        try:
440            # May raise `FTPIOError` if directory isn't writable.
441            file_ = self.open(helper_file_name, "w")
442            file_.close()
443        except ftputil.error.FTPIOError:
444            raise ftputil.error.TimeShiftError(
445                """couldn't write helper file in directory '{}'""".format(self.getcwd())
446            )
447        # If everything worked up to here it should be possible to stat and
448        # then remove the just-written file.
449        try:
450            server_time = self.path.getmtime(helper_file_name)
451            self.unlink(helper_file_name)
452        except ftputil.error.FTPOSError:
453            # If we got a `TimeShiftError` exception above, we should't come
454            # here: if we didn't get a `TimeShiftError` above, deletion should
455            # be possible. The only reason for an exception I can think of here
456            # is a race condition by removing the helper file or write
457            # permission from the directory or helper file after it has been
458            # written to.
459            raise ftputil.error.TimeShiftError(
460                "could write helper file but not unlink it"
461            )
462        # Calculate the difference between server and client.
463        now = time.time()
464        time_shift = server_time - now
465        # As the time shift for this host instance isn't set yet, the directory
466        # parser will calculate times one year in the past if the time zone of
467        # the server is east from ours. Thus the time shift will be off by a
468        # year as well (see ticket #55).
469        if time_shift < -360 * 24 * 60 * 60:
470            # Read one year and recalculate the time shift. We don't know how
471            # many days made up that year (it might have been a leap year), so
472            # go the route via `datetime.replace`.
473            server_datetime = datetime.datetime.fromtimestamp(
474                server_time, tz=datetime.timezone.utc
475            )
476            server_datetime = server_datetime.replace(year=server_datetime.year + 1)
477            time_shift = server_datetime.timestamp() - now
478        self.set_time_shift(time_shift)
479
480    #
481    # Operations based on file-like objects (rather high-level), like upload
482    # and download
483    #
484    # XXX: This has a different API from `shutil.copyfileobj`, on which this
485    # method is modeled. But I don't think it makes sense to change this method
486    # here because the method is probably rarely used and a change would break
487    # client code.
488    @staticmethod
489    def copyfileobj(
490        source,
491        target,
492        max_chunk_size=ftputil.file_transfer.MAX_COPY_CHUNK_SIZE,
493        callback=None,
494    ):
495        """
496        Copy data from file-like object `source` to file-like object `target`.
497        """
498        ftputil.file_transfer.copyfileobj(source, target, max_chunk_size, callback)
499
500    def _upload_files(self, source_path, target_path):
501        """
502        Return a `LocalFile` and `RemoteFile` as source and target,
503        respectively.
504
505        The strings `source_path` and `target_path` are the (absolute or
506        relative) paths of the local and the remote file, respectively.
507        """
508        source_file = ftputil.file_transfer.LocalFile(source_path, "rb")
509        # Passing `self` (the `FTPHost` instance) here is correct.
510        target_file = ftputil.file_transfer.RemoteFile(self, target_path, "wb")
511        return source_file, target_file
512
513    def upload(self, source, target, callback=None):
514        """
515        Upload a file from the local source (name) to the remote target (name).
516
517        If a callable `callback` is given, it's called after every chunk of
518        transferred data. The chunk size is a constant defined in
519        `file_transfer`. The callback will be called with a single argument,
520        the data chunk that was transferred before the callback was called.
521        """
522        target = ftputil.tool.as_str_path(target, encoding=self._encoding)
523        source_file, target_file = self._upload_files(source, target)
524        ftputil.file_transfer.copy_file(
525            source_file, target_file, conditional=False, callback=callback
526        )
527
528    def upload_if_newer(self, source, target, callback=None):
529        """
530        Upload a file only if it's newer than the target on the remote host or
531        if the target file does not exist. See the method `upload` for the
532        meaning of the parameters.
533
534        If an upload was necessary, return `True`, else return `False`.
535
536        If a callable `callback` is given, it's called after every chunk of
537        transferred data. The chunk size is a constant defined in
538        `file_transfer`. The callback will be called with a single argument,
539        the data chunk that was transferred before the callback was called.
540        """
541        target = ftputil.tool.as_str_path(target, encoding=self._encoding)
542        source_file, target_file = self._upload_files(source, target)
543        return ftputil.file_transfer.copy_file(
544            source_file, target_file, conditional=True, callback=callback
545        )
546
547    def _download_files(self, source_path, target_path):
548        """
549        Return a `RemoteFile` and `LocalFile` as source and target,
550        respectively.
551
552        The strings `source_path` and `target_path` are the (absolute or
553        relative) paths of the remote and the local file, respectively.
554        """
555        source_file = ftputil.file_transfer.RemoteFile(self, source_path, "rb")
556        target_file = ftputil.file_transfer.LocalFile(target_path, "wb")
557        return source_file, target_file
558
559    def download(self, source, target, callback=None):
560        """
561        Download a file from the remote source (name) to the local target
562        (name).
563
564        If a callable `callback` is given, it's called after every chunk of
565        transferred data. The chunk size is a constant defined in
566        `file_transfer`. The callback will be called with a single argument,
567        the data chunk that was transferred before the callback was called.
568        """
569        source = ftputil.tool.as_str_path(source, encoding=self._encoding)
570        source_file, target_file = self._download_files(source, target)
571        ftputil.file_transfer.copy_file(
572            source_file, target_file, conditional=False, callback=callback
573        )
574
575    def download_if_newer(self, source, target, callback=None):
576        """
577        Download a file only if it's newer than the target on the local host or
578        if the target file does not exist. See the method `download` for the
579        meaning of the parameters.
580
581        If a download was necessary, return `True`, else return `False`.
582
583        If a callable `callback` is given, it's called after every chunk of
584        transferred data. The chunk size is a constant defined in
585        `file_transfer`. The callback will be called with a single argument,
586        the data chunk that was transferred before the callback was called.
587        """
588        source = ftputil.tool.as_str_path(source, encoding=self._encoding)
589        source_file, target_file = self._download_files(source, target)
590        return ftputil.file_transfer.copy_file(
591            source_file, target_file, conditional=True, callback=callback
592        )
593
594    #
595    # Helper methods to descend into a directory before executing a command
596    #
597    def _check_inaccessible_login_directory(self):
598        """
599        Raise an `InaccessibleLoginDirError` exception if we can't change to
600        the login directory. This test is only reliable if the current
601        directory is the login directory.
602        """
603        presumable_login_dir = self.getcwd()
604        # Bail out with an internal error rather than modify the current
605        # directory without hope of restoration.
606        try:
607            self.chdir(presumable_login_dir)
608        except ftputil.error.PermanentError:
609            raise ftputil.error.InaccessibleLoginDirError(
610                "directory '{}' is not accessible".format(presumable_login_dir)
611            )
612
613    def _robust_ftp_command(self, command, path, descend_deeply=False):
614        """
615        Run an FTP command on a path. The return value of the method is the
616        return value of the command.
617
618        If `descend_deeply` is true (the default is false), descend deeply,
619        i. e. change the directory to the end of the path.
620        """
621        # If we can't change to the yet-current directory, the code below won't
622        # work (see below), so in this case rather raise an exception than
623        # giving wrong results.
624        self._check_inaccessible_login_directory()
625        # Some FTP servers don't behave as expected if the directory portion of
626        # the path contains whitespace; some even yield strange results if the
627        # command isn't executed in the current directory. Therefore, change to
628        # the directory which contains the item to run the command on and
629        # invoke the command just there.
630        #
631        # Remember old working directory.
632        old_dir = self.getcwd()
633        try:
634            if descend_deeply:
635                # Invoke the command in (not: on) the deepest directory.
636                self.chdir(path)
637                # Workaround for some servers that give recursive listings when
638                # called with a dot as path; see issue #33,
639                # http://ftputil.sschwarzer.net/trac/ticket/33
640                return command(self, "")
641            else:
642                # Invoke the command in the "next to last" directory.
643                head, tail = self.path.split(path)
644                self.chdir(head)
645                return command(self, tail)
646        finally:
647            self.chdir(old_dir)
648
649    #
650    # Miscellaneous utility methods resembling functions in `os`
651    #
652    def getcwd(self):
653        """
654        Return the current path name.
655        """
656        return self._cached_current_dir
657
658    def chdir(self, path):
659        """
660        Change the directory on the host.
661        """
662        path = ftputil.tool.as_str_path(path, encoding=self._encoding)
663        with ftputil.error.ftplib_error_to_ftp_os_error:
664            self._session.cwd(path)
665        # The path given as the argument is relative to the old current
666        # directory, therefore join them.
667        self._cached_current_dir = self.path.normpath(
668            self.path.join(self._cached_current_dir, path)
669        )
670
671    # Ignore unused argument `mode`
672    # pylint: disable=unused-argument
673    def mkdir(self, path, mode=None):
674        """
675        Make the directory path on the remote host. The argument `mode` is
676        ignored and only "supported" for similarity with `os.mkdir`.
677        """
678        path = ftputil.tool.as_str_path(path, encoding=self._encoding)
679
680        def command(self, path):
681            """Callback function."""
682            with ftputil.error.ftplib_error_to_ftp_os_error:
683                self._session.mkd(path)
684
685        self._robust_ftp_command(command, path)
686
687    # TODO: The virtual directory support doesn't have unit tests yet because
688    # the mocking most likely would be quite complicated. The tests should be
689    # added when mainly the `mock` library is used instead of the mock code in
690    # `test.mock_ftplib`.
691    #
692    # Ignore unused argument `mode`
693    # pylint: disable=unused-argument
694    def makedirs(self, path, mode=None, exist_ok=False):
695        """
696        Make the directory `path`, but also make not yet existing intermediate
697        directories, like `os.makedirs`.
698
699        The value of `mode` is only accepted for compatibility with
700        `os.makedirs` but otherwise ignored.
701
702        If `exist_ok` is `False` (the default) and the leaf directory exists,
703        raise a `PermanentError` with `errno` 17.
704        """
705        path = ftputil.tool.as_str_path(path, encoding=self._encoding)
706        path = self.path.abspath(path)
707        directories = path.split(self.sep)
708        old_dir = self.getcwd()
709        try:
710            # Try to build the directory chain from the "uppermost" to the
711            # "lowermost" directory.
712            for index in range(1, len(directories)):
713                # Re-insert the separator which got lost by using `path.split`.
714                next_directory = self.sep + self.path.join(*directories[: index + 1])
715                # If we have "virtual directories" (see #86), just listing the
716                # parent directory won't tell us if a directory actually
717                # exists. So try to change into the directory.
718                try:
719                    self.chdir(next_directory)
720                except ftputil.error.PermanentError:
721                    # Directory presumably doesn't exist.
722                    try:
723                        self.mkdir(next_directory)
724                    except ftputil.error.PermanentError:
725                        # Find out the cause of the error. Re-raise the
726                        # exception only if the directory didn't exist already,
727                        # else something went _really_ wrong, e. g. there's a
728                        # regular file with the name of the directory.
729                        if not self.path.isdir(next_directory):
730                            raise
731                else:
732                    # Directory exists. If we are at the last directory
733                    # component and `exist_ok` is `False`, this is an error.
734                    if (index == len(directories) - 1) and (not exist_ok):
735                        # Before PEP 3151, if `exist_ok` is `False`, trying to
736                        # create an existing directory in the local file system
737                        # results in an `OSError` with `errno.EEXIST, so
738                        # emulate this also for FTP.
739                        ftp_os_error = ftputil.error.PermanentError(
740                            "path {!r} exists".format(path)
741                        )
742                        ftp_os_error.errno = errno.EEXIST
743                        raise ftp_os_error
744        finally:
745            self.chdir(old_dir)
746
747    def rmdir(self, path):
748        """
749        Remove the _empty_ directory `path` on the remote host.
750
751        Compatibility note:
752
753        Previous versions of ftputil could possibly delete non-empty
754        directories as well, - if the server allowed it. This is no longer
755        supported.
756        """
757        path = ftputil.tool.as_str_path(path, encoding=self._encoding)
758        path = self.path.abspath(path)
759        if self.listdir(path):
760            raise ftputil.error.PermanentError("directory '{}' not empty".format(path))
761        # XXX: How does `rmd` work with links?
762        def command(self, path):
763            """Callback function."""
764            with ftputil.error.ftplib_error_to_ftp_os_error:
765                self._session.rmd(path)
766
767        self._robust_ftp_command(command, path)
768        self.stat_cache.invalidate(path)
769
770    def remove(self, path):
771        """
772        Remove the file or link given by `path`.
773
774        Raise a `PermanentError` if the path doesn't exist, but maybe raise
775        other exceptions depending on the state of the server (e. g. timeout).
776        """
777        path = ftputil.tool.as_str_path(path, encoding=self._encoding)
778        path = self.path.abspath(path)
779        # Though `isfile` includes also links to files, `islink` is needed to
780        # include links to directories.
781        if (
782            self.path.isfile(path)
783            or self.path.islink(path)
784            or not self.path.exists(path)
785        ):
786            # If the path doesn't exist, let the removal command trigger an
787            # exception with a more appropriate error message.
788            def command(self, path):
789                """Callback function."""
790                with ftputil.error.ftplib_error_to_ftp_os_error:
791                    self._session.delete(path)
792
793            self._robust_ftp_command(command, path)
794        else:
795            raise ftputil.error.PermanentError(
796                "remove/unlink can only delete files and links, " "not directories"
797            )
798        self.stat_cache.invalidate(path)
799
800    unlink = remove
801
802    def rmtree(self, path, ignore_errors=False, onerror=None):
803        """
804        Remove the given remote, possibly non-empty, directory tree. The
805        interface of this method is rather complex, in favor of compatibility
806        with `shutil.rmtree`.
807
808        If `ignore_errors` is set to a true value, errors are ignored. If
809        `ignore_errors` is a false value _and_ `onerror` isn't set, all
810        exceptions occurring during the tree iteration and processing are
811        raised. These exceptions are all of type `PermanentError`.
812
813        To distinguish between error situations, pass in a callable for
814        `onerror`. This callable must accept three arguments: `func`, `path`
815        and `exc_info`. `func` is a bound method object, _for example_
816        `your_host_object.listdir`. `path` is the path that was the recent
817        argument of the respective method (`listdir`, `remove`, `rmdir`).
818        `exc_info` is the exception info as it's got from `sys.exc_info`.
819
820        Implementation note: The code is copied from `shutil.rmtree` in
821        Python 2.4 and adapted to ftputil.
822        """
823        path = ftputil.tool.as_str_path(path, encoding=self._encoding)
824        # The following code is an adapted version of Python 2.4's
825        # `shutil.rmtree` function.
826        if ignore_errors:
827
828            def new_onerror(*args):
829                """Do nothing."""
830                # pylint: disable=unused-argument
831                pass
832
833        elif onerror is None:
834
835            def new_onerror(*args):
836                """Re-raise exception."""
837                # pylint: disable=misplaced-bare-raise, unused-argument
838                raise
839
840        else:
841            new_onerror = onerror
842        names = []
843        try:
844            names = self.listdir(path)
845        except ftputil.error.PermanentError:
846            new_onerror(self.listdir, path, sys.exc_info())
847        for name in names:
848            full_name = self.path.join(path, name)
849            try:
850                mode = self.lstat(full_name).st_mode
851            except ftputil.error.PermanentError:
852                mode = 0
853            if stat.S_ISDIR(mode):
854                self.rmtree(full_name, ignore_errors, new_onerror)
855            else:
856                try:
857                    self.remove(full_name)
858                except ftputil.error.PermanentError:
859                    new_onerror(self.remove, full_name, sys.exc_info())
860        try:
861            self.rmdir(path)
862        except ftputil.error.FTPOSError:
863            new_onerror(self.rmdir, path, sys.exc_info())
864
865    def rename(self, source, target):
866        """
867        Rename the source on the FTP host to target.
868        """
869        source = ftputil.tool.as_str_path(source, encoding=self._encoding)
870        target = ftputil.tool.as_str_path(target, encoding=self._encoding)
871        # The following code is in spirit similar to the code in the method
872        # `_robust_ftp_command`, though we do _not_ do _everything_ imaginable.
873        self._check_inaccessible_login_directory()
874        source_head, source_tail = self.path.split(source)
875        target_head, target_tail = self.path.split(target)
876        paths_contain_whitespace = (" " in source_head) or (" " in target_head)
877        if paths_contain_whitespace and source_head == target_head:
878            # Both items are in the same directory.
879            old_dir = self.getcwd()
880            try:
881                self.chdir(source_head)
882                with ftputil.error.ftplib_error_to_ftp_os_error:
883                    self._session.rename(source_tail, target_tail)
884            finally:
885                self.chdir(old_dir)
886        else:
887            # Use straightforward command.
888            with ftputil.error.ftplib_error_to_ftp_os_error:
889                self._session.rename(source, target)
890
891    # XXX: One could argue to put this method into the `_Stat` class, but I
892    # refrained from that because then `_Stat` would have to know about
893    # `FTPHost`'s `_session` attribute and in turn about `_session`'s `dir`
894    # method.
895    def _dir(self, path):
896        """
897        Return a directory listing as made by FTP's `LIST` command as a list of
898        strings.
899        """
900        # Don't use `self.path.isdir` in this method because that would cause a
901        # call of `(l)stat` and thus a call to `_dir`, so we would end up with
902        # an infinite recursion.
903        def _FTPHost_dir_command(self, path):
904            """Callback function."""
905            lines = []
906
907            def callback(line):
908                """Callback function."""
909                lines.append(ftputil.tool.as_str(line, encoding=self._encoding))
910
911            with ftputil.error.ftplib_error_to_ftp_os_error:
912                if self.use_list_a_option:
913                    self._session.dir("-a", path, callback)
914                else:
915                    self._session.dir(path, callback)
916            return lines
917
918        lines = self._robust_ftp_command(
919            _FTPHost_dir_command, path, descend_deeply=True
920        )
921        return lines
922
923    # The `listdir`, `lstat` and `stat` methods don't use `_robust_ftp_command`
924    # because they implicitly already use `_dir` which actually uses
925    # `_robust_ftp_command`.
926
927    def listdir(self, path):
928        """
929        Return a list of directories, files etc. in the directory named `path`.
930
931        If the directory listing from the server can't be parsed with any of
932        the available parsers raise a `ParserError`.
933        """
934        original_path = path
935        path = ftputil.tool.as_str_path(path, encoding=self._encoding)
936        items = self._stat._listdir(path)
937        return [
938            ftputil.tool.same_string_type_as(original_path, item, self._encoding)
939            for item in items
940        ]
941
942    def lstat(self, path, _exception_for_missing_path=True):
943        """
944        Return an object similar to that returned by `os.lstat`.
945
946        If the directory listing from the server can't be parsed with any of
947        the available parsers, raise a `ParserError`. If the directory _can_ be
948        parsed and the `path` is _not_ found, raise a `PermanentError`.
949
950        (`_exception_for_missing_path` is an implementation aid and _not_
951        intended for use by ftputil clients.)
952        """
953        path = ftputil.tool.as_str_path(path, encoding=self._encoding)
954        return self._stat._lstat(path, _exception_for_missing_path)
955
956    def stat(self, path, _exception_for_missing_path=True):
957        """
958        Return info from a "stat" call on `path`.
959
960        If the directory containing `path` can't be parsed, raise a
961        `ParserError`. If the directory containing `path` can be parsed but the
962        `path` can't be found, raise a `PermanentError`. Also raise a
963        `PermanentError` if there's an endless (cyclic) chain of symbolic links
964        "behind" the `path`.
965
966        (`_exception_for_missing_path` is an implementation aid and _not_
967        intended for use by ftputil clients.)
968        """
969        path = ftputil.tool.as_str_path(path, encoding=self._encoding)
970        return self._stat._stat(path, _exception_for_missing_path)
971
972    def walk(self, top, topdown=True, onerror=None, followlinks=False):
973        """
974        Iterate over directory tree and return a tuple (dirpath, dirnames,
975        filenames) on each iteration, like the `os.walk` function (see
976        https://docs.python.org/library/os.html#os.walk ).
977        """
978        top = ftputil.tool.as_str_path(top, encoding=self._encoding)
979        # The following code is copied from `os.walk` in Python 2.4 and adapted
980        # to ftputil.
981        try:
982            names = self.listdir(top)
983        except ftputil.error.FTPOSError as err:
984            if onerror is not None:
985                onerror(err)
986            return
987        dirs, nondirs = [], []
988        for name in names:
989            if self.path.isdir(self.path.join(top, name)):
990                dirs.append(name)
991            else:
992                nondirs.append(name)
993        if topdown:
994            yield top, dirs, nondirs
995        for name in dirs:
996            path = self.path.join(top, name)
997            if followlinks or not self.path.islink(path):
998                yield from self.walk(path, topdown, onerror, followlinks)
999        if not topdown:
1000            yield top, dirs, nondirs
1001
1002    def chmod(self, path, mode):
1003        """
1004        Change the mode of a remote `path` (a string) to the integer `mode`.
1005        This integer uses the same bits as the mode value returned by the
1006        `stat` and `lstat` commands.
1007
1008        If something goes wrong, raise a `TemporaryError` or a
1009        `PermanentError`, according to the status code returned by the server.
1010        In particular, a non-existent path usually causes a `PermanentError`.
1011        """
1012        path = ftputil.tool.as_str_path(path, encoding=self._encoding)
1013        path = self.path.abspath(path)
1014
1015        def command(self, path):
1016            """Callback function."""
1017            with ftputil.error.ftplib_error_to_ftp_os_error:
1018                self._session.voidcmd("SITE CHMOD 0{0:o} {1}".format(mode, path))
1019
1020        self._robust_ftp_command(command, path)
1021        self.stat_cache.invalidate(path)
1022
1023    def __getstate__(self):
1024        raise TypeError("cannot serialize FTPHost object")
1025
1026    #
1027    # Context manager methods
1028    #
1029    def __enter__(self):
1030        # Return `self`, so it can be accessed as the variable component of the
1031        # `with` statement.
1032        return self
1033
1034    def __exit__(self, exc_type, exc_val, exc_tb):
1035        # We don't need the `exc_*` arguments here.
1036        # pylint: disable=unused-argument
1037        self.close()
1038        # Be explicit.
1039        return False