source: ftputil/host.py @ 1925:68fd6debe77c

Last change on this file since 1925:68fd6debe77c was 1925:68fd6debe77c, checked in by Stefan Schwarzer <sschwarzer@…>, 18 months ago
Clear cache if time shift changes If the time shift changes, all timestamps in the cache will be wrong with respect to the _new_ time shift. Depending on the cache, reusing the old timestamps may not be a problem, but in some cases it might be. Therefore stay on the safe side and clear the cache, sacrificing efficiency in favor of correct timestamps. At first sight, we could "correct" the cache entries by applying the difference between old and new time shift to the existing cache entries. However, I think the interactions in ftputil are already so complicated that I'd like to avoid any probably unnecessary optimizations. Also, the time shift would probably be set soon after creating the `FTPHost` instance, so anyway only relatively few cache entries would be affected.
File size: 39.8 KB
Line 
1# Copyright (C) 2002-2020, Stefan Schwarzer <sschwarzer@sschwarzer.net>
2# and ftputil contributors (see `doc/contributors.txt`)
3# See the file LICENSE for licensing terms.
4
5"""
6`FTPHost` is the central class of the `ftputil` library.
7
8See `__init__.py` for an example.
9"""
10
11import ftplib
12import stat
13import sys
14import time
15
16import ftputil.error
17import ftputil.file
18import ftputil.file_transfer
19import ftputil.path
20import ftputil.stat
21import ftputil.tool
22
23
24__all__ = ["FTPHost"]
25
26
27# The "protected" attributes PyLint talks about aren't intended for
28# clients of the library. `FTPHost` objects need to use some of these
29# library-internal attributes though.
30# pylint: disable=protected-access
31
32
33#####################################################################
34# `FTPHost` class with several methods similar to those of `os`
35
36
37class FTPHost:
38    """FTP host class."""
39
40    # Implementation notes:
41    #
42    # Upon every request of a file (`FTPFile` object) a new FTP
43    # session is created (or reused from a cache), leading to a child
44    # session of the `FTPHost` object from which the file is requested.
45    #
46    # This is needed because opening an `FTPFile` will make the
47    # local session object wait for the completion of the transfer.
48    # In fact, code like this would block indefinitely, if the `RETR`
49    # request would be made on the `_session` of the object host:
50    #
51    #   host = FTPHost(ftp_server, user, password)
52    #   f = host.open("index.html")
53    #   host.getcwd()   # would block!
54    #
55    # On the other hand, the initially constructed host object will
56    # store references to already established `FTPFile` objects and
57    # reuse an associated connection if its associated `FTPFile`
58    # has been closed.
59
60    def __init__(self, *args, **kwargs):
61        """Abstract initialization of `FTPHost` object."""
62        # Store arguments for later operations.
63        self._args = args
64        self._kwargs = kwargs
65        # XXX: Maybe put the following in a `reset` method.
66        # The time shift setting shouldn't be reset though.
67        # Make a session according to these arguments.
68        self._session = self._make_session()
69        # Simulate `os.path`.
70        self.path = ftputil.path._Path(self)
71        # lstat, stat, listdir services.
72        self._stat = ftputil.stat._Stat(self)
73        self.stat_cache = self._stat._lstat_cache
74        self.stat_cache.enable()
75        with ftputil.error.ftplib_error_to_ftp_os_error:
76            self._cached_current_dir = self.path.normpath(
77                ftputil.tool.as_str_path(self._session.pwd())
78            )
79        # Associated `FTPHost` objects for data transfer.
80        self._children = []
81        # This is only set to something else than `None` if this
82        # instance represents an `FTPFile`.
83        self._file = None
84        # Now opened.
85        self.closed = False
86        # Set curdir, pardir etc. for the remote host. RFC 959 states
87        # that this is, strictly speaking, dependent on the server OS
88        # but it seems to work at least with Unix and Windows servers.
89        self.curdir, self.pardir, self.sep = ".", "..", "/"
90        # Set default time shift (used in `upload_if_newer` and
91        # `download_if_newer`).
92        self._time_shift = 0.0
93        # Don't use `LIST -a` option by default. If the server doesn't
94        # understand the `-a` option and interprets it as a path, the
95        # results can be surprising. See ticket #110.
96        self.use_list_a_option = False
97
98    def keep_alive(self):
99        """
100        Try to keep the connection alive in order to avoid server timeouts.
101
102        Note that this won't help if the connection has already timed
103        out! In this case, `keep_alive` will raise an `TemporaryError`.
104        (Actually, if you get a server timeout, the error - for a specific
105        connection - will be permanent.)
106        """
107        # Warning: Don't call this method on `FTPHost` instances which
108        # represent file transfers. This may fail in confusing ways.
109        with ftputil.error.ftplib_error_to_ftp_os_error:
110            # Ignore return value.
111            self._session.pwd()
112
113    #
114    # Dealing with child sessions and file-like objects
115    # (rather low-level)
116    #
117    def _make_session(self):
118        """
119        Return a new session object according to the current state of
120        this `FTPHost` instance.
121        """
122        # Don't modify original attributes below.
123        args = self._args[:]
124        kwargs = self._kwargs.copy()
125        # If a session factory has been given on the instantiation of
126        # this `FTPHost` object, use the same factory for this
127        # `FTPHost` object's child sessions.
128        factory = kwargs.pop("session_factory", ftplib.FTP)
129        with ftputil.error.ftplib_error_to_ftp_os_error:
130            session = factory(*args, **kwargs)
131        return session
132
133    def _copy(self):
134        """Return a copy of this `FTPHost` object."""
135        # The copy includes a new session factory return value (aka
136        # session) but doesn't copy the state of `self.getcwd()`.
137        return self.__class__(*self._args, **self._kwargs)
138
139    def _available_child(self):
140        """
141        Return an available (i. e. one whose `_file` object is closed
142        and doesn't have a timed-out server connection) child
143        (`FTPHost` object) from the pool of children or `None` if
144        there aren't any.
145        """
146        # TODO: Currently timed-out child sessions aren't removed and
147        # may collect over time. In very busy or long running
148        # processes, this might slow down an application because the
149        # same stale child sessions have to be processed again and
150        # again.
151        for host in self._children:
152            # Test for timeouts only after testing for a closed file:
153            # - If a file isn't closed, save time; don't bother to access
154            #   the remote server.
155            # - If a file transfer on the child is in progress, requesting
156            #   the directory is an invalid operation because of the way
157            #   the FTP state machine works (see RFC 959).
158            if host._file.closed:
159                try:
160                    host._session.pwd()
161                # Under high load, a 226 status response from a
162                # previous download may arrive too late, so that it's
163                # "seen" in the `pwd` call. For now, skip the
164                # potential child session; it will be considered again
165                # when `_available_child` is called the next time.
166                except ftplib.error_reply:
167                    continue
168                # Timed-out sessions raise `error_temp`.
169                except ftplib.error_temp:
170                    continue
171                # The server may have closed the connection which may
172                # cause `host._session.getline` to raise an `EOFError`
173                # (see ticket #114).
174                except EOFError:
175                    continue
176                # Under high load, there may be a socket read timeout
177                # during the last FTP file `close` (see ticket #112).
178                # Note that a socket timeout is quite different from
179                # an FTP session timeout.
180                except OSError:
181                    continue
182                else:
183                    # Everything's ok; use this `FTPHost` instance.
184                    return host
185        # Be explicit.
186        return None
187
188    def open(
189        self,
190        path,
191        mode="r",
192        buffering=None,
193        encoding=None,
194        errors=None,
195        newline=None,
196        *,
197        rest=None,
198    ):
199        """
200        Return an open file(-like) object which is associated with
201        this `FTPHost` object.
202
203        The arguments `path`, `mode`, `buffering`, `encoding`,
204        `errors` and `newlines` have the same meaning as for `open`.
205        If `rest` is given as an integer,
206
207        - reading will start at the byte (zero-based) `rest`
208        - writing will overwrite the remote file from byte `rest`
209
210        This method tries to reuse a child but will generate a new one
211        if none is available.
212        """
213        # Support the same arguments as `open`.
214        # pylint: disable=too-many-arguments
215        path = ftputil.tool.as_str_path(path)
216        host = self._available_child()
217        if host is None:
218            host = self._copy()
219            self._children.append(host)
220            host._file = ftputil.file.FTPFile(host)
221        basedir = self.getcwd()
222        # Prepare for changing the directory (see whitespace workaround
223        # in method `_dir`).
224        if host.path.isabs(path):
225            effective_path = path
226        else:
227            effective_path = host.path.join(basedir, path)
228        effective_dir, effective_file = host.path.split(effective_path)
229        try:
230            # This will fail if the directory isn't accessible at all.
231            host.chdir(effective_dir)
232        except ftputil.error.PermanentError:
233            # Similarly to a failed `file` in a local file system,
234            # raise an `IOError`, not an `OSError`.
235            raise ftputil.error.FTPIOError(
236                "remote directory '{}' doesn't "
237                "exist or has insufficient access rights".format(effective_dir)
238            )
239        host._file._open(
240            effective_file,
241            mode=mode,
242            buffering=buffering,
243            encoding=encoding,
244            errors=errors,
245            newline=newline,
246            rest=rest,
247        )
248        if "w" in mode:
249            # Invalidate cache entry because size and timestamps will change.
250            self.stat_cache.invalidate(effective_path)
251        return host._file
252
253    def close(self):
254        """Close host connection."""
255        if self.closed:
256            return
257        # Close associated children.
258        for host in self._children:
259            # Children have a `_file` attribute which is an `FTPFile` object.
260            host._file.close()
261            host.close()
262        # Now deal with ourself.
263        try:
264            with ftputil.error.ftplib_error_to_ftp_os_error:
265                self._session.close()
266        finally:
267            # If something went wrong before, the host/session is
268            # probably defunct and subsequent calls to `close` won't
269            # help either, so consider the host/session closed for
270            # practical purposes.
271            self.stat_cache.clear()
272            self._children = []
273            self.closed = True
274
275    #
276    # Setting a custom directory parser
277    #
278    def set_parser(self, parser):
279        """
280        Set the parser for extracting stat results from directory
281        listings.
282
283        The parser interface is described in the documentation, but
284        here are the most important things:
285
286        - A parser should derive from `ftputil.stat.Parser`.
287
288        - The parser has to implement two methods, `parse_line` and
289          `ignores_line`. For the latter, there's a probably useful
290          default in the class `ftputil.stat.Parser`.
291
292        - `parse_line` should try to parse a line of a directory
293          listing and return a `ftputil.stat.StatResult` instance. If
294          parsing isn't possible, raise `ftputil.error.ParserError`
295          with a useful error message.
296
297        - `ignores_line` should return a true value if the line isn't
298          assumed to contain stat information.
299        """
300        # The cache contents, if any, probably aren't useful.
301        self.stat_cache.clear()
302        # Set the parser explicitly, don't allow "smart" switching anymore.
303        self._stat._parser = parser
304        self._stat._allow_parser_switching = False
305
306    #
307    # Time shift adjustment between client (i. e. us) and server
308    #
309    def set_time_shift(self, time_shift):
310        """
311        Set the time shift value (i. e. the time difference between
312        client and server, where the client time is in UTC) for this
313        `FTPHost` object. By (my) definition, the time shift value is
314        positive if the local time of the server is greater than the
315        local time of the client (for the same physical time), i. e.
316
317            time_shift =def= t_server - t_client_utc
318        <=> t_server = t_client_utc + time_shift
319        <=> t_client_utc = t_server - time_shift
320
321        The time shift is measured in seconds.
322        """
323        old_time_shift = self.time_shift()
324        if time_shift != old_time_shift:
325            # If the time shift changed, all entries in the cache will have
326            # wrong times with respect to the updated time shift, therefore
327            # clear the cache.
328            self.stat_cache.clear()
329            self._time_shift = time_shift
330
331    def time_shift(self):
332        """
333        Return the time shift between FTP server and client. See the
334        docstring of `set_time_shift` for more on this value.
335        """
336        return self._time_shift
337
338    @staticmethod
339    def __rounded_time_shift(time_shift):
340        """
341        Return the given time shift in seconds, but rounded to
342        15-minute units. The argument is also assumed to be given in
343        seconds.
344        """
345        minute = 60.0
346        # Avoid division by zero below.
347        if time_shift == 0:
348            return 0.0
349        # Use a positive value for rounding.
350        absolute_time_shift = abs(time_shift)
351        signum = time_shift / absolute_time_shift
352        # Round absolute time shift to 15-minute units.
353        absolute_rounded_time_shift = int(
354            (absolute_time_shift + (7.5 * minute)) / (15.0 * minute)
355        ) * (15.0 * minute)
356        # Return with correct sign.
357        return signum * absolute_rounded_time_shift
358
359    def __assert_valid_time_shift(self, time_shift):
360        """
361        Perform sanity checks on the time shift value (given in
362        seconds). If the value is invalid, raise a `TimeShiftError`,
363        else simply return `None`.
364        """
365        minute = 60.0  # seconds
366        hour = 60.0 * minute
367        absolute_rounded_time_shift = abs(self.__rounded_time_shift(time_shift))
368        # Test 1: Fail if the absolute time shift is greater than
369        #         a full day (24 hours).
370        if absolute_rounded_time_shift > 24 * hour:
371            raise ftputil.error.TimeShiftError(
372                "time shift abs({0:.2f} s) > 1 day".format(time_shift)
373            )
374        # Test 2: Fail if the deviation between given time shift and
375        #         15-minute units is greater than a certain limit.
376        maximum_deviation = 5 * minute
377        if abs(time_shift - self.__rounded_time_shift(time_shift)) > maximum_deviation:
378            raise ftputil.error.TimeShiftError(
379                "time shift ({0:.2f} s) deviates more than {1:d} s "
380                "from 15-minute units".format(time_shift, int(maximum_deviation))
381            )
382
383    def synchronize_times(self):
384        """
385        Synchronize the local times of FTP client and server. This
386        is necessary to let `upload_if_newer` and `download_if_newer`
387        work correctly. If `synchronize_times` isn't applicable
388        (see below), the time shift can still be set explicitly with
389        `set_time_shift`.
390
391        This implementation of `synchronize_times` requires _all_ of
392        the following:
393
394        - The connection between server and client is established.
395        - The client has write access to the directory that is
396          current when `synchronize_times` is called.
397
398        The common usage pattern of `synchronize_times` is to call it
399        directly after the connection is established. (As can be
400        concluded from the points above, this requires write access
401        to the login directory.)
402
403        If `synchronize_times` fails, it raises a `TimeShiftError`.
404        """
405        helper_file_name = "_ftputil_sync_"
406        # Open a dummy file for writing in the current directory
407        # on the FTP host, then close it.
408        try:
409            # May raise `FTPIOError` if directory isn't writable.
410            file_ = self.open(helper_file_name, "w")
411            file_.close()
412        except ftputil.error.FTPIOError:
413            raise ftputil.error.TimeShiftError(
414                """couldn't write helper file in directory '{}'""".format(self.getcwd())
415            )
416        # If everything worked up to here it should be possible to stat
417        # and then remove the just-written file.
418        try:
419            server_time = self.path.getmtime(helper_file_name)
420            self.unlink(helper_file_name)
421        except ftputil.error.FTPOSError:
422            # If we got a `TimeShiftError` exception above, we
423            # should't come here: if we didn't get a `TimeShiftError`
424            # above, deletion should be possible. The only reason for
425            # an exception I can think of here is a race condition by
426            # removing the helper file or write permission from the
427            # directory or helper file after it has been written to.
428            raise ftputil.error.TimeShiftError(
429                "could write helper file but not unlink it"
430            )
431        # Calculate the difference between server and client.
432        now = time.time()
433        time_shift = server_time - now
434        # As the time shift for this host instance isn't set yet, the
435        # directory parser will calculate times one year in the past if
436        # the time zone of the server is east from ours. Thus the time
437        # shift will be off by a year as well (see ticket #55).
438        if time_shift < -360 * 24 * 60 * 60:
439            # Re-add one year and re-calculate the time shift. We don't
440            # know how many days made up that year (it might have been
441            # a leap year), so go the route via `time.localtime` and
442            # `time.mktime`.
443            server_time_struct = time.localtime(server_time)
444            server_time_struct = (server_time_struct.tm_year + 1,) + server_time_struct[
445                1:
446            ]
447            server_time = time.mktime(server_time_struct)
448            time_shift = server_time - now
449        # Do some sanity checks.
450        self.__assert_valid_time_shift(time_shift)
451        # If tests passed, store the time difference as time shift value.
452        self.set_time_shift(self.__rounded_time_shift(time_shift))
453
454    #
455    # Operations based on file-like objects (rather high-level),
456    # like upload and download
457    #
458    # XXX: This has a different API from `shutil.copyfileobj`, on which this
459    # method is modeled. But I don't think it makes sense to change this method
460    # here because the method is probably rarely used and a change would break
461    # client code.
462    @staticmethod
463    def copyfileobj(
464        source,
465        target,
466        max_chunk_size=ftputil.file_transfer.MAX_COPY_CHUNK_SIZE,
467        callback=None,
468    ):
469        """
470        Copy data from file-like object `source` to file-like object
471        `target`.
472        """
473        ftputil.file_transfer.copyfileobj(source, target, max_chunk_size, callback)
474
475    def _upload_files(self, source_path, target_path):
476        """
477        Return a `LocalFile` and `RemoteFile` as source and target,
478        respectively.
479
480        The strings `source_path` and `target_path` are the (absolute
481        or relative) paths of the local and the remote file, respectively.
482        """
483        source_file = ftputil.file_transfer.LocalFile(source_path, "rb")
484        # Passing `self` (the `FTPHost` instance) here is correct.
485        target_file = ftputil.file_transfer.RemoteFile(self, target_path, "wb")
486        return source_file, target_file
487
488    def upload(self, source, target, callback=None):
489        """
490        Upload a file from the local source (name) to the remote
491        target (name).
492
493        If a callable `callback` is given, it's called after every
494        chunk of transferred data. The chunk size is a constant
495        defined in `file_transfer`. The callback will be called with a
496        single argument, the data chunk that was transferred before
497        the callback was called.
498        """
499        target = ftputil.tool.as_str_path(target)
500        source_file, target_file = self._upload_files(source, target)
501        ftputil.file_transfer.copy_file(
502            source_file, target_file, conditional=False, callback=callback
503        )
504
505    def upload_if_newer(self, source, target, callback=None):
506        """
507        Upload a file only if it's newer than the target on the
508        remote host or if the target file does not exist. See the
509        method `upload` for the meaning of the parameters.
510
511        If an upload was necessary, return `True`, else return `False`.
512
513        If a callable `callback` is given, it's called after every
514        chunk of transferred data. The chunk size is a constant
515        defined in `file_transfer`. The callback will be called with a
516        single argument, the data chunk that was transferred before
517        the callback was called.
518        """
519        target = ftputil.tool.as_str_path(target)
520        source_file, target_file = self._upload_files(source, target)
521        return ftputil.file_transfer.copy_file(
522            source_file, target_file, conditional=True, callback=callback
523        )
524
525    def _download_files(self, source_path, target_path):
526        """
527        Return a `RemoteFile` and `LocalFile` as source and target,
528        respectively.
529
530        The strings `source_path` and `target_path` are the (absolute
531        or relative) paths of the remote and the local file, respectively.
532        """
533        source_file = ftputil.file_transfer.RemoteFile(self, source_path, "rb")
534        target_file = ftputil.file_transfer.LocalFile(target_path, "wb")
535        return source_file, target_file
536
537    def download(self, source, target, callback=None):
538        """
539        Download a file from the remote source (name) to the local
540        target (name).
541
542        If a callable `callback` is given, it's called after every
543        chunk of transferred data. The chunk size is a constant
544        defined in `file_transfer`. The callback will be called with a
545        single argument, the data chunk that was transferred before
546        the callback was called.
547        """
548        source = ftputil.tool.as_str_path(source)
549        source_file, target_file = self._download_files(source, target)
550        ftputil.file_transfer.copy_file(
551            source_file, target_file, conditional=False, callback=callback
552        )
553
554    def download_if_newer(self, source, target, callback=None):
555        """
556        Download a file only if it's newer than the target on the
557        local host or if the target file does not exist. See the
558        method `download` for the meaning of the parameters.
559
560        If a download was necessary, return `True`, else return
561        `False`.
562
563        If a callable `callback` is given, it's called after every
564        chunk of transferred data. The chunk size is a constant
565        defined in `file_transfer`. The callback will be called with a
566        single argument, the data chunk that was transferred before
567        the callback was called.
568        """
569        source = ftputil.tool.as_str_path(source)
570        source_file, target_file = self._download_files(source, target)
571        return ftputil.file_transfer.copy_file(
572            source_file, target_file, conditional=True, callback=callback
573        )
574
575    #
576    # Helper methods to descend into a directory before executing a command
577    #
578    def _check_inaccessible_login_directory(self):
579        """
580        Raise an `InaccessibleLoginDirError` exception if we can't
581        change to the login directory. This test is only reliable if
582        the current directory is the login directory.
583        """
584        presumable_login_dir = self.getcwd()
585        # Bail out with an internal error rather than modify the
586        # current directory without hope of restoration.
587        try:
588            self.chdir(presumable_login_dir)
589        except ftputil.error.PermanentError:
590            raise ftputil.error.InaccessibleLoginDirError(
591                "directory '{}' is not accessible".format(presumable_login_dir)
592            )
593
594    def _robust_ftp_command(self, command, path, descend_deeply=False):
595        """
596        Run an FTP command on a path. The return value of the method
597        is the return value of the command.
598
599        If `descend_deeply` is true (the default is false), descend
600        deeply, i. e. change the directory to the end of the path.
601        """
602        # If we can't change to the yet-current directory, the code
603        # below won't work (see below), so in this case rather raise
604        # an exception than giving wrong results.
605        self._check_inaccessible_login_directory()
606        # Some FTP servers don't behave as expected if the directory
607        # portion of the path contains whitespace; some even yield
608        # strange results if the command isn't executed in the
609        # current directory. Therefore, change to the directory
610        # which contains the item to run the command on and invoke
611        # the command just there.
612        #
613        # Remember old working directory.
614        old_dir = self.getcwd()
615        try:
616            if descend_deeply:
617                # Invoke the command in (not: on) the deepest directory.
618                self.chdir(path)
619                # Workaround for some servers that give recursive
620                # listings when called with a dot as path; see issue #33,
621                # http://ftputil.sschwarzer.net/trac/ticket/33
622                return command(self, "")
623            else:
624                # Invoke the command in the "next to last" directory.
625                head, tail = self.path.split(path)
626                self.chdir(head)
627                return command(self, tail)
628        finally:
629            self.chdir(old_dir)
630
631    #
632    # Miscellaneous utility methods resembling functions in `os`
633    #
634    def getcwd(self):
635        """Return the current path name."""
636        return self._cached_current_dir
637
638    def chdir(self, path):
639        """Change the directory on the host."""
640        path = ftputil.tool.as_str_path(path)
641        with ftputil.error.ftplib_error_to_ftp_os_error:
642            self._session.cwd(path)
643        # The path given as the argument is relative to the old current
644        # directory, therefore join them.
645        self._cached_current_dir = self.path.normpath(
646            self.path.join(self._cached_current_dir, path)
647        )
648
649    # Ignore unused argument `mode`
650    # pylint: disable=unused-argument
651    def mkdir(self, path, mode=None):
652        """
653        Make the directory path on the remote host. The argument
654        `mode` is ignored and only "supported" for similarity with
655        `os.mkdir`.
656        """
657        path = ftputil.tool.as_str_path(path)
658
659        def command(self, path):
660            """Callback function."""
661            with ftputil.error.ftplib_error_to_ftp_os_error:
662                self._session.mkd(path)
663
664        self._robust_ftp_command(command, path)
665
666    # TODO: The virtual directory support doesn't have unit tests yet
667    # because the mocking most likely would be quite complicated. The
668    # tests should be added when mainly the `mock` library is used
669    # instead of the mock code in `test.mock_ftplib`.
670    #
671    # Ignore unused argument `mode`
672    # pylint: disable=unused-argument
673    def makedirs(self, path, mode=None):
674        """
675        Make the directory `path`, but also make not yet existing
676        intermediate directories, like `os.makedirs`. The value of
677        `mode` is only accepted for compatibility with `os.makedirs`
678        but otherwise ignored.
679        """
680        path = ftputil.tool.as_str_path(path)
681        path = self.path.abspath(path)
682        directories = path.split(self.sep)
683        old_dir = self.getcwd()
684        try:
685            # Try to build the directory chain from the "uppermost" to
686            # the "lowermost" directory.
687            for index in range(1, len(directories)):
688                # Re-insert the separator which got lost by using
689                # `path.split`.
690                next_directory = self.sep + self.path.join(*directories[: index + 1])
691                # If we have "virtual directories" (see #86), just
692                # listing the parent directory won't tell us if a
693                # directory actually exists. So try to change into the
694                # directory.
695                try:
696                    self.chdir(next_directory)
697                except ftputil.error.PermanentError:
698                    try:
699                        self.mkdir(next_directory)
700                    except ftputil.error.PermanentError:
701                        # Find out the cause of the error. Re-raise
702                        # the exception only if the directory didn't
703                        # exist already, else something went _really_
704                        # wrong, e. g. there's a regular file with the
705                        # name of the directory.
706                        if not self.path.isdir(next_directory):
707                            raise
708        finally:
709            self.chdir(old_dir)
710
711    def rmdir(self, path):
712        """
713        Remove the _empty_ directory `path` on the remote host.
714
715        Compatibility note:
716
717        Previous versions of ftputil could possibly delete non-
718        empty directories as well, - if the server allowed it. This
719        is no longer supported.
720        """
721        path = ftputil.tool.as_str_path(path)
722        path = self.path.abspath(path)
723        if self.listdir(path):
724            raise ftputil.error.PermanentError("directory '{}' not empty".format(path))
725        # XXX: How does `rmd` work with links?
726        def command(self, path):
727            """Callback function."""
728            with ftputil.error.ftplib_error_to_ftp_os_error:
729                self._session.rmd(path)
730
731        self._robust_ftp_command(command, path)
732        self.stat_cache.invalidate(path)
733
734    def remove(self, path):
735        """
736        Remove the file or link given by `path`.
737
738        Raise a `PermanentError` if the path doesn't exist, but maybe
739        raise other exceptions depending on the state of the server
740        (e. g. timeout).
741        """
742        path = ftputil.tool.as_str_path(path)
743        path = self.path.abspath(path)
744        # Though `isfile` includes also links to files, `islink`
745        # is needed to include links to directories.
746        if (
747            self.path.isfile(path)
748            or self.path.islink(path)
749            or not self.path.exists(path)
750        ):
751            # If the path doesn't exist, let the removal command trigger
752            # an exception with a more appropriate error message.
753            def command(self, path):
754                """Callback function."""
755                with ftputil.error.ftplib_error_to_ftp_os_error:
756                    self._session.delete(path)
757
758            self._robust_ftp_command(command, path)
759        else:
760            raise ftputil.error.PermanentError(
761                "remove/unlink can only delete files and links, " "not directories"
762            )
763        self.stat_cache.invalidate(path)
764
765    unlink = remove
766
767    def rmtree(self, path, ignore_errors=False, onerror=None):
768        """
769        Remove the given remote, possibly non-empty, directory tree.
770        The interface of this method is rather complex, in favor of
771        compatibility with `shutil.rmtree`.
772
773        If `ignore_errors` is set to a true value, errors are ignored.
774        If `ignore_errors` is a false value _and_ `onerror` isn't set,
775        all exceptions occurring during the tree iteration and
776        processing are raised. These exceptions are all of type
777        `PermanentError`.
778
779        To distinguish between error situations, pass in a callable
780        for `onerror`. This callable must accept three arguments:
781        `func`, `path` and `exc_info`. `func` is a bound method
782        object, _for example_ `your_host_object.listdir`. `path` is
783        the path that was the recent argument of the respective method
784        (`listdir`, `remove`, `rmdir`). `exc_info` is the exception
785        info as it's got from `sys.exc_info`.
786
787        Implementation note: The code is copied from `shutil.rmtree`
788        in Python 2.4 and adapted to ftputil.
789        """
790        path = ftputil.tool.as_str_path(path)
791        # The following code is an adapted version of Python 2.4's
792        # `shutil.rmtree` function.
793        if ignore_errors:
794
795            def new_onerror(*args):
796                """Do nothing."""
797                # pylint: disable=unused-argument
798                pass
799
800        elif onerror is None:
801
802            def new_onerror(*args):
803                """Re-raise exception."""
804                # pylint: disable=misplaced-bare-raise, unused-argument
805                raise
806
807        else:
808            new_onerror = onerror
809        names = []
810        try:
811            names = self.listdir(path)
812        except ftputil.error.PermanentError:
813            new_onerror(self.listdir, path, sys.exc_info())
814        for name in names:
815            full_name = self.path.join(path, name)
816            try:
817                mode = self.lstat(full_name).st_mode
818            except ftputil.error.PermanentError:
819                mode = 0
820            if stat.S_ISDIR(mode):
821                self.rmtree(full_name, ignore_errors, new_onerror)
822            else:
823                try:
824                    self.remove(full_name)
825                except ftputil.error.PermanentError:
826                    new_onerror(self.remove, full_name, sys.exc_info())
827        try:
828            self.rmdir(path)
829        except ftputil.error.FTPOSError:
830            new_onerror(self.rmdir, path, sys.exc_info())
831
832    def rename(self, source, target):
833        """Rename the source on the FTP host to target."""
834        source = ftputil.tool.as_str_path(source)
835        target = ftputil.tool.as_str_path(target)
836        # The following code is in spirit similar to the code in the
837        # method `_robust_ftp_command`, though we do _not_ do
838        # _everything_ imaginable.
839        self._check_inaccessible_login_directory()
840        source_head, source_tail = self.path.split(source)
841        target_head, target_tail = self.path.split(target)
842        paths_contain_whitespace = (" " in source_head) or (" " in target_head)
843        if paths_contain_whitespace and source_head == target_head:
844            # Both items are in the same directory.
845            old_dir = self.getcwd()
846            try:
847                self.chdir(source_head)
848                with ftputil.error.ftplib_error_to_ftp_os_error:
849                    self._session.rename(source_tail, target_tail)
850            finally:
851                self.chdir(old_dir)
852        else:
853            # Use straightforward command.
854            with ftputil.error.ftplib_error_to_ftp_os_error:
855                self._session.rename(source, target)
856
857    # XXX: One could argue to put this method into the `_Stat` class,
858    # but I refrained from that because then `_Stat` would have to
859    # know about `FTPHost`'s `_session` attribute and in turn about
860    # `_session`'s `dir` method.
861    def _dir(self, path):
862        """
863        Return a directory listing as made by FTP's `LIST` command as
864        a list of strings.
865        """
866        # Don't use `self.path.isdir` in this method because that
867        # would cause a call of `(l)stat` and thus a call to `_dir`,
868        # so we would end up with an infinite recursion.
869        def _FTPHost_dir_command(self, path):
870            """Callback function."""
871            lines = []
872
873            def callback(line):
874                """Callback function."""
875                lines.append(ftputil.tool.as_str(line))
876
877            with ftputil.error.ftplib_error_to_ftp_os_error:
878                if self.use_list_a_option:
879                    self._session.dir("-a", path, callback)
880                else:
881                    self._session.dir(path, callback)
882            return lines
883
884        lines = self._robust_ftp_command(
885            _FTPHost_dir_command, path, descend_deeply=True
886        )
887        return lines
888
889    # The `listdir`, `lstat` and `stat` methods don't use
890    # `_robust_ftp_command` because they implicitly already use
891    # `_dir` which actually uses `_robust_ftp_command`.
892
893    def listdir(self, path):
894        """
895        Return a list of directories, files etc. in the directory
896        named `path`.
897
898        If the directory listing from the server can't be parsed with
899        any of the available parsers raise a `ParserError`.
900        """
901        original_path = path
902        path = ftputil.tool.as_str_path(path)
903        items = self._stat._listdir(path)
904        return [ftputil.tool.same_string_type_as(original_path, item) for item in items]
905
906    def lstat(self, path, _exception_for_missing_path=True):
907        """
908        Return an object similar to that returned by `os.lstat`.
909
910        If the directory listing from the server can't be parsed with
911        any of the available parsers, raise a `ParserError`. If the
912        directory _can_ be parsed and the `path` is _not_ found, raise
913        a `PermanentError`.
914
915        (`_exception_for_missing_path` is an implementation aid and
916        _not_ intended for use by ftputil clients.)
917        """
918        path = ftputil.tool.as_str_path(path)
919        return self._stat._lstat(path, _exception_for_missing_path)
920
921    def stat(self, path, _exception_for_missing_path=True):
922        """
923        Return info from a "stat" call on `path`.
924
925        If the directory containing `path` can't be parsed, raise a
926        `ParserError`. If the directory containing `path` can be
927        parsed but the `path` can't be found, raise a
928        `PermanentError`. Also raise a `PermanentError` if there's an
929        endless (cyclic) chain of symbolic links "behind" the `path`.
930
931        (`_exception_for_missing_path` is an implementation aid and
932        _not_ intended for use by ftputil clients.)
933        """
934        path = ftputil.tool.as_str_path(path)
935        return self._stat._stat(path, _exception_for_missing_path)
936
937    def walk(self, top, topdown=True, onerror=None, followlinks=False):
938        """
939        Iterate over directory tree and return a tuple (dirpath,
940        dirnames, filenames) on each iteration, like the `os.walk`
941        function (see https://docs.python.org/library/os.html#os.walk ).
942        """
943        top = ftputil.tool.as_str_path(top)
944        # The following code is copied from `os.walk` in Python 2.4
945        # and adapted to ftputil.
946        try:
947            names = self.listdir(top)
948        except ftputil.error.FTPOSError as err:
949            if onerror is not None:
950                onerror(err)
951            return
952        dirs, nondirs = [], []
953        for name in names:
954            if self.path.isdir(self.path.join(top, name)):
955                dirs.append(name)
956            else:
957                nondirs.append(name)
958        if topdown:
959            yield top, dirs, nondirs
960        for name in dirs:
961            path = self.path.join(top, name)
962            if followlinks or not self.path.islink(path):
963                yield from self.walk(path, topdown, onerror, followlinks)
964        if not topdown:
965            yield top, dirs, nondirs
966
967    def chmod(self, path, mode):
968        """
969        Change the mode of a remote `path` (a string) to the integer
970        `mode`. This integer uses the same bits as the mode value
971        returned by the `stat` and `lstat` commands.
972
973        If something goes wrong, raise a `TemporaryError` or a
974        `PermanentError`, according to the status code returned by
975        the server. In particular, a non-existent path usually
976        causes a `PermanentError`.
977        """
978        path = ftputil.tool.as_str_path(path)
979        path = self.path.abspath(path)
980
981        def command(self, path):
982            """Callback function."""
983            with ftputil.error.ftplib_error_to_ftp_os_error:
984                self._session.voidcmd("SITE CHMOD 0{0:o} {1}".format(mode, path))
985
986        self._robust_ftp_command(command, path)
987        self.stat_cache.invalidate(path)
988
989    def __getstate__(self):
990        raise TypeError("cannot serialize FTPHost object")
991
992    #
993    # Context manager methods
994    #
995    def __enter__(self):
996        # Return `self`, so it can be accessed as the variable
997        # component of the `with` statement.
998        return self
999
1000    def __exit__(self, exc_type, exc_val, exc_tb):
1001        # We don't need the `exc_*` arguments here.
1002        # pylint: disable=unused-argument
1003        self.close()
1004        # Be explicit.
1005        return False
Note: See TracBrowser for help on using the repository browser.