source: ftputil/host.py

Last change on this file was 1939:bc64a54e638c, checked in by Stefan Schwarzer <sschwarzer@…>, 5 weeks ago
Fix definition of time shift value This has changed with the implemenation of ticket #134, see https://ftputil.sschwarzer.net/trac/ticket/134 . ticket: 134
File size: 40.3 KB
Line 
1# Copyright (C) 2002-2020, Stefan Schwarzer <sschwarzer@sschwarzer.net>
2# and ftputil contributors (see `doc/contributors.txt`)
3# See the file LICENSE for licensing terms.
4
5"""
6`FTPHost` is the central class of the `ftputil` library.
7
8See `__init__.py` for an example.
9"""
10
11import datetime
12import errno
13import ftplib
14import stat
15import sys
16import time
17
18import ftputil.error
19import ftputil.file
20import ftputil.file_transfer
21import ftputil.path
22import ftputil.stat
23import ftputil.tool
24
25
26__all__ = ["FTPHost"]
27
28
29# The "protected" attributes PyLint talks about aren't intended for clients of
30# the library. `FTPHost` objects need to use some of these library-internal
31# attributes though.
32# pylint: disable=protected-access
33
34
35#####################################################################
36# `FTPHost` class with several methods similar to those of `os`
37
38
39class FTPHost:
40    """
41    FTP host class.
42    """
43
44    # Implementation notes:
45    #
46    # Upon every request of a file (`FTPFile` object) a new FTP session is
47    # created (or reused from a cache), leading to a child session of the
48    # `FTPHost` object from which the file is requested.
49    #
50    # This is needed because opening an `FTPFile` will make the local session
51    # object wait for the completion of the transfer. In fact, code like this
52    # would block indefinitely, if the `RETR` request would be made on the
53    # `_session` of the object host:
54    #
55    #   host = FTPHost(ftp_server, user, password)
56    #   f = host.open("index.html")
57    #   host.getcwd()   # would block!
58    #
59    # On the other hand, the initially constructed host object will store
60    # references to already established `FTPFile` objects and reuse an
61    # associated connection if its associated `FTPFile` has been closed.
62
63    def __init__(self, *args, **kwargs):
64        """
65        Abstract initialization of `FTPHost` object.
66        """
67        # Store arguments for later operations.
68        self._args = args
69        self._kwargs = kwargs
70        # XXX: Maybe put the following in a `reset` method.
71        # The time shift setting shouldn't be reset though. Make a session
72        # according to these arguments.
73        self._session = self._make_session()
74        # Simulate `os.path`.
75        self.path = ftputil.path._Path(self)
76        # lstat, stat, listdir services.
77        self._stat = ftputil.stat._Stat(self)
78        self.stat_cache = self._stat._lstat_cache
79        self.stat_cache.enable()
80        with ftputil.error.ftplib_error_to_ftp_os_error:
81            self._cached_current_dir = self.path.normpath(
82                ftputil.tool.as_str_path(self._session.pwd())
83            )
84        # Associated `FTPHost` objects for data transfer.
85        self._children = []
86        # This is only set to something else than `None` if this instance
87        # represents an `FTPFile`.
88        self._file = None
89        # Now opened.
90        self.closed = False
91        # Set curdir, pardir etc. for the remote host. RFC 959 states that this
92        # is, strictly speaking, dependent on the server OS but it seems to
93        # work at least with Unix and Windows servers.
94        self.curdir, self.pardir, self.sep = ".", "..", "/"
95        # Set default time shift (used in `upload_if_newer` and
96        # `download_if_newer`).
97        self._time_shift = 0.0
98        # Don't use `LIST -a` option by default. If the server doesn't
99        # understand the `-a` option and interprets it as a path, the results
100        # can be surprising. See ticket #110.
101        self.use_list_a_option = False
102
103    def keep_alive(self):
104        """
105        Try to keep the connection alive in order to avoid server timeouts.
106
107        Note that this won't help if the connection has already timed out! In
108        this case, `keep_alive` will raise an `TemporaryError`. (Actually, if
109        you get a server timeout, the error - for a specific connection - will
110        be permanent.)
111        """
112        # Warning: Don't call this method on `FTPHost` instances which
113        # represent file transfers. This may fail in confusing ways.
114        with ftputil.error.ftplib_error_to_ftp_os_error:
115            # Ignore return value.
116            self._session.pwd()
117
118    #
119    # Dealing with child sessions and file-like objects (rather low-level)
120    #
121    def _make_session(self):
122        """
123        Return a new session object according to the current state of this
124        `FTPHost` instance.
125        """
126        # Don't modify original attributes below.
127        args = self._args[:]
128        kwargs = self._kwargs.copy()
129        # If a session factory has been given on the instantiation of this
130        # `FTPHost` object, use the same factory for this `FTPHost` object's
131        # child sessions.
132        factory = kwargs.pop("session_factory", ftplib.FTP)
133        with ftputil.error.ftplib_error_to_ftp_os_error:
134            session = factory(*args, **kwargs)
135        return session
136
137    def _copy(self):
138        """
139        Return a copy of this `FTPHost` object.
140        """
141        # The copy includes a new session factory return value (aka session)
142        # but doesn't copy the state of `self.getcwd()`.
143        return self.__class__(*self._args, **self._kwargs)
144
145    def _available_child(self):
146        """
147        Return an available (i. e. one whose `_file` object is closed and
148        doesn't have a timed-out server connection) child (`FTPHost` object)
149        from the pool of children or `None` if there aren't any.
150        """
151        # TODO: Currently timed-out child sessions aren't removed and may
152        # collect over time. In very busy or long running processes, this might
153        # slow down an application because the same stale child sessions have
154        # to be processed again and again.
155        for host in self._children:
156            # Test for timeouts only after testing for a closed file:
157            # - If a file isn't closed, save time; don't bother to access the
158            #   remote server.
159            # - If a file transfer on the child is in progress, requesting the
160            #   directory is an invalid operation because of the way the FTP
161            #   state machine works (see RFC 959).
162            if host._file.closed:
163                try:
164                    host._session.pwd()
165                # Under high load, a 226 status response from a previous
166                # download may arrive too late, so that it's "seen" in the
167                # `pwd` call. For now, skip the potential child session; it
168                # will be considered again when `_available_child` is called
169                # the next time.
170                except ftplib.error_reply:
171                    continue
172                # Timed-out sessions raise `error_temp`.
173                except ftplib.error_temp:
174                    continue
175                # The server may have closed the connection which may cause
176                # `host._session.getline` to raise an `EOFError` (see ticket
177                # #114).
178                except EOFError:
179                    continue
180                # Under high load, there may be a socket read timeout during
181                # the last FTP file `close` (see ticket #112). Note that a
182                # socket timeout is quite different from an FTP session
183                # timeout.
184                except OSError:
185                    continue
186                else:
187                    # Everything's ok; use this `FTPHost` instance.
188                    return host
189        # Be explicit.
190        return None
191
192    def open(
193        self,
194        path,
195        mode="r",
196        buffering=None,
197        encoding=None,
198        errors=None,
199        newline=None,
200        *,
201        rest=None,
202    ):
203        """
204        Return an open file(-like) object which is associated with this
205        `FTPHost` object.
206
207        The arguments `path`, `mode`, `buffering`, `encoding`, `errors` and
208        `newlines` have the same meaning as for `open`. If `rest` is given as
209        an integer,
210
211        - reading will start at the byte (zero-based) `rest`
212        - writing will overwrite the remote file from byte `rest`
213
214        This method tries to reuse a child but will generate a new one if none
215        is available.
216        """
217        # Support the same arguments as `open`.
218        # pylint: disable=too-many-arguments
219        path = ftputil.tool.as_str_path(path)
220        host = self._available_child()
221        if host is None:
222            host = self._copy()
223            self._children.append(host)
224            host._file = ftputil.file.FTPFile(host)
225        basedir = self.getcwd()
226        # Prepare for changing the directory (see whitespace workaround in
227        # method `_dir`).
228        if host.path.isabs(path):
229            effective_path = path
230        else:
231            effective_path = host.path.join(basedir, path)
232        effective_dir, effective_file = host.path.split(effective_path)
233        try:
234            # This will fail if the directory isn't accessible at all.
235            host.chdir(effective_dir)
236        except ftputil.error.PermanentError:
237            # Similarly to a failed `file` in a local file system, raise an
238            # `IOError`, not an `OSError`.
239            raise ftputil.error.FTPIOError(
240                "remote directory '{}' doesn't "
241                "exist or has insufficient access rights".format(effective_dir)
242            )
243        host._file._open(
244            effective_file,
245            mode=mode,
246            buffering=buffering,
247            encoding=encoding,
248            errors=errors,
249            newline=newline,
250            rest=rest,
251        )
252        if "w" in mode:
253            # Invalidate cache entry because size and timestamps will change.
254            self.stat_cache.invalidate(effective_path)
255        return host._file
256
257    def close(self):
258        """
259        Close host connection.
260        """
261        if self.closed:
262            return
263        # Close associated children.
264        for host in self._children:
265            # Children have a `_file` attribute which is an `FTPFile` object.
266            host._file.close()
267            host.close()
268        # Now deal with ourself.
269        try:
270            with ftputil.error.ftplib_error_to_ftp_os_error:
271                self._session.close()
272        finally:
273            # If something went wrong before, the host/session is probably
274            # defunct and subsequent calls to `close` won't help either, so
275            # consider the host/session closed for practical purposes.
276            self.stat_cache.clear()
277            self._children = []
278            self.closed = True
279
280    #
281    # Setting a custom directory parser
282    #
283    def set_parser(self, parser):
284        """
285        Set the parser for extracting stat results from directory listings.
286
287        The parser interface is described in the documentation, but here are
288        the most important things:
289
290        - A parser should derive from `ftputil.stat.Parser`.
291
292        - The parser has to implement two methods, `parse_line` and
293          `ignores_line`. For the latter, there's a probably useful default
294          in the class `ftputil.stat.Parser`.
295
296        - `parse_line` should try to parse a line of a directory listing and
297          return a `ftputil.stat.StatResult` instance. If parsing isn't
298          possible, raise `ftputil.error.ParserError` with a useful error
299          message.
300
301        - `ignores_line` should return a true value if the line isn't assumed
302          to contain stat information.
303        """
304        # The cache contents, if any, probably aren't useful.
305        self.stat_cache.clear()
306        # Set the parser explicitly, don't allow "smart" switching anymore.
307        self._stat._parser = parser
308        self._stat._allow_parser_switching = False
309
310    #
311    # Time shift adjustment between client (i. e. us) and server
312    #
313    @staticmethod
314    def __rounded_time_shift(time_shift):
315        """
316        Return the given time shift in seconds, but rounded to 15-minute units.
317        The argument is also assumed to be given in seconds.
318        """
319        minute = 60.0
320        # Avoid division by zero below.
321        if time_shift == 0:
322            return 0.0
323        # Use a positive value for rounding.
324        absolute_time_shift = abs(time_shift)
325        signum = time_shift / absolute_time_shift
326        # Round absolute time shift to 15-minute units.
327        absolute_rounded_time_shift = int(
328            (absolute_time_shift + (7.5 * minute)) / (15.0 * minute)
329        ) * (15.0 * minute)
330        # Return with correct sign.
331        return signum * absolute_rounded_time_shift
332
333    def __assert_valid_time_shift(self, time_shift):
334        """
335        Perform sanity checks on the time shift value (given in seconds). If
336        the value is invalid, raise a `TimeShiftError`, else simply return
337        `None`.
338        """
339        minute = 60.0  # seconds
340        hour = 60.0 * minute
341        absolute_rounded_time_shift = abs(self.__rounded_time_shift(time_shift))
342        # Test 1: Fail if the absolute time shift is greater than a full day
343        #         (24 hours).
344        if absolute_rounded_time_shift > 24 * hour:
345            raise ftputil.error.TimeShiftError(
346                "time shift abs({0:.2f} s) > 1 day".format(time_shift)
347            )
348        # Test 2: Fail if the deviation between given time shift and 15-minute
349        #         units is greater than a certain limit.
350        maximum_deviation = 5 * minute
351        if abs(time_shift - self.__rounded_time_shift(time_shift)) > maximum_deviation:
352            raise ftputil.error.TimeShiftError(
353                "time shift ({0:.2f} s) deviates more than {1:d} s "
354                "from 15-minute units".format(time_shift, int(maximum_deviation))
355            )
356
357    def set_time_shift(self, time_shift):
358        """
359        Set the time shift value.
360
361        By (my) definition, the time shift value is the difference of
362        the time zone used in server listings and UTC, i. e.
363
364            time_shift =def= t_server - utc
365        <=> t_server = utc + time_shift
366        <=> utc = t_server - time_shift
367
368        The time shift is measured in seconds.
369        """
370        self.__assert_valid_time_shift(time_shift)
371        old_time_shift = self.time_shift()
372        if time_shift != old_time_shift:
373            # If the time shift changed, all entries in the cache will have
374            # wrong times with respect to the updated time shift, therefore
375            # clear the cache.
376            self.stat_cache.clear()
377            self._time_shift = self.__rounded_time_shift(time_shift)
378
379    def time_shift(self):
380        """
381        Return the time shift between FTP server and client. See the
382        docstring of `set_time_shift` for more on this value.
383        """
384        return self._time_shift
385
386    def synchronize_times(self):
387        """
388        Synchronize the local times of FTP client and server. This is necessary
389        to let `upload_if_newer` and `download_if_newer` work correctly. If
390        `synchronize_times` isn't applicable (see below), the time shift can
391        still be set explicitly with `set_time_shift`.
392
393        This implementation of `synchronize_times` requires _all_ of the
394        following:
395
396        - The connection between server and client is established.
397        - The client has write access to the directory that is current when
398          `synchronize_times` is called.
399
400        The common usage pattern of `synchronize_times` is to call it directly
401        after the connection is established. (As can be concluded from the
402        points above, this requires write access to the login directory.)
403
404        If `synchronize_times` fails, it raises a `TimeShiftError`.
405        """
406        helper_file_name = "_ftputil_sync_"
407        # Open a dummy file for writing in the current directory on the FTP
408        # host, then close it.
409        try:
410            # May raise `FTPIOError` if directory isn't writable.
411            file_ = self.open(helper_file_name, "w")
412            file_.close()
413        except ftputil.error.FTPIOError:
414            raise ftputil.error.TimeShiftError(
415                """couldn't write helper file in directory '{}'""".format(self.getcwd())
416            )
417        # If everything worked up to here it should be possible to stat and
418        # then remove the just-written file.
419        try:
420            server_time = self.path.getmtime(helper_file_name)
421            self.unlink(helper_file_name)
422        except ftputil.error.FTPOSError:
423            # If we got a `TimeShiftError` exception above, we should't come
424            # here: if we didn't get a `TimeShiftError` above, deletion should
425            # be possible. The only reason for an exception I can think of here
426            # is a race condition by removing the helper file or write
427            # permission from the directory or helper file after it has been
428            # written to.
429            raise ftputil.error.TimeShiftError(
430                "could write helper file but not unlink it"
431            )
432        # Calculate the difference between server and client.
433        now = time.time()
434        time_shift = server_time - now
435        # As the time shift for this host instance isn't set yet, the directory
436        # parser will calculate times one year in the past if the time zone of
437        # the server is east from ours. Thus the time shift will be off by a
438        # year as well (see ticket #55).
439        if time_shift < -360 * 24 * 60 * 60:
440            # Read one year and recalculate the time shift. We don't know how
441            # many days made up that year (it might have been a leap year), so
442            # go the route via `datetime.replace`.
443            server_datetime = datetime.datetime.fromtimestamp(
444                server_time, tz=datetime.timezone.utc
445            )
446            server_datetime = server_datetime.replace(year=server_datetime.year + 1)
447            time_shift = server_datetime.timestamp() - now
448        self.set_time_shift(time_shift)
449
450    #
451    # Operations based on file-like objects (rather high-level), like upload
452    # and download
453    #
454    # XXX: This has a different API from `shutil.copyfileobj`, on which this
455    # method is modeled. But I don't think it makes sense to change this method
456    # here because the method is probably rarely used and a change would break
457    # client code.
458    @staticmethod
459    def copyfileobj(
460        source,
461        target,
462        max_chunk_size=ftputil.file_transfer.MAX_COPY_CHUNK_SIZE,
463        callback=None,
464    ):
465        """
466        Copy data from file-like object `source` to file-like object `target`.
467        """
468        ftputil.file_transfer.copyfileobj(source, target, max_chunk_size, callback)
469
470    def _upload_files(self, source_path, target_path):
471        """
472        Return a `LocalFile` and `RemoteFile` as source and target,
473        respectively.
474
475        The strings `source_path` and `target_path` are the (absolute or
476        relative) paths of the local and the remote file, respectively.
477        """
478        source_file = ftputil.file_transfer.LocalFile(source_path, "rb")
479        # Passing `self` (the `FTPHost` instance) here is correct.
480        target_file = ftputil.file_transfer.RemoteFile(self, target_path, "wb")
481        return source_file, target_file
482
483    def upload(self, source, target, callback=None):
484        """
485        Upload a file from the local source (name) to the remote target (name).
486
487        If a callable `callback` is given, it's called after every chunk of
488        transferred data. The chunk size is a constant defined in
489        `file_transfer`. The callback will be called with a single argument,
490        the data chunk that was transferred before the callback was called.
491        """
492        target = ftputil.tool.as_str_path(target)
493        source_file, target_file = self._upload_files(source, target)
494        ftputil.file_transfer.copy_file(
495            source_file, target_file, conditional=False, callback=callback
496        )
497
498    def upload_if_newer(self, source, target, callback=None):
499        """
500        Upload a file only if it's newer than the target on the remote host or
501        if the target file does not exist. See the method `upload` for the
502        meaning of the parameters.
503
504        If an upload was necessary, return `True`, else return `False`.
505
506        If a callable `callback` is given, it's called after every chunk of
507        transferred data. The chunk size is a constant defined in
508        `file_transfer`. The callback will be called with a single argument,
509        the data chunk that was transferred before the callback was called.
510        """
511        target = ftputil.tool.as_str_path(target)
512        source_file, target_file = self._upload_files(source, target)
513        return ftputil.file_transfer.copy_file(
514            source_file, target_file, conditional=True, callback=callback
515        )
516
517    def _download_files(self, source_path, target_path):
518        """
519        Return a `RemoteFile` and `LocalFile` as source and target,
520        respectively.
521
522        The strings `source_path` and `target_path` are the (absolute or
523        relative) paths of the remote and the local file, respectively.
524        """
525        source_file = ftputil.file_transfer.RemoteFile(self, source_path, "rb")
526        target_file = ftputil.file_transfer.LocalFile(target_path, "wb")
527        return source_file, target_file
528
529    def download(self, source, target, callback=None):
530        """
531        Download a file from the remote source (name) to the local target
532        (name).
533
534        If a callable `callback` is given, it's called after every chunk of
535        transferred data. The chunk size is a constant defined in
536        `file_transfer`. The callback will be called with a single argument,
537        the data chunk that was transferred before the callback was called.
538        """
539        source = ftputil.tool.as_str_path(source)
540        source_file, target_file = self._download_files(source, target)
541        ftputil.file_transfer.copy_file(
542            source_file, target_file, conditional=False, callback=callback
543        )
544
545    def download_if_newer(self, source, target, callback=None):
546        """
547        Download a file only if it's newer than the target on the local host or
548        if the target file does not exist. See the method `download` for the
549        meaning of the parameters.
550
551        If a download was necessary, return `True`, else return `False`.
552
553        If a callable `callback` is given, it's called after every chunk of
554        transferred data. The chunk size is a constant defined in
555        `file_transfer`. The callback will be called with a single argument,
556        the data chunk that was transferred before the callback was called.
557        """
558        source = ftputil.tool.as_str_path(source)
559        source_file, target_file = self._download_files(source, target)
560        return ftputil.file_transfer.copy_file(
561            source_file, target_file, conditional=True, callback=callback
562        )
563
564    #
565    # Helper methods to descend into a directory before executing a command
566    #
567    def _check_inaccessible_login_directory(self):
568        """
569        Raise an `InaccessibleLoginDirError` exception if we can't change to
570        the login directory. This test is only reliable if the current
571        directory is the login directory.
572        """
573        presumable_login_dir = self.getcwd()
574        # Bail out with an internal error rather than modify the current
575        # directory without hope of restoration.
576        try:
577            self.chdir(presumable_login_dir)
578        except ftputil.error.PermanentError:
579            raise ftputil.error.InaccessibleLoginDirError(
580                "directory '{}' is not accessible".format(presumable_login_dir)
581            )
582
583    def _robust_ftp_command(self, command, path, descend_deeply=False):
584        """
585        Run an FTP command on a path. The return value of the method is the
586        return value of the command.
587
588        If `descend_deeply` is true (the default is false), descend deeply,
589        i. e. change the directory to the end of the path.
590        """
591        # If we can't change to the yet-current directory, the code below won't
592        # work (see below), so in this case rather raise an exception than
593        # giving wrong results.
594        self._check_inaccessible_login_directory()
595        # Some FTP servers don't behave as expected if the directory portion of
596        # the path contains whitespace; some even yield strange results if the
597        # command isn't executed in the current directory. Therefore, change to
598        # the directory which contains the item to run the command on and
599        # invoke the command just there.
600        #
601        # Remember old working directory.
602        old_dir = self.getcwd()
603        try:
604            if descend_deeply:
605                # Invoke the command in (not: on) the deepest directory.
606                self.chdir(path)
607                # Workaround for some servers that give recursive listings when
608                # called with a dot as path; see issue #33,
609                # http://ftputil.sschwarzer.net/trac/ticket/33
610                return command(self, "")
611            else:
612                # Invoke the command in the "next to last" directory.
613                head, tail = self.path.split(path)
614                self.chdir(head)
615                return command(self, tail)
616        finally:
617            self.chdir(old_dir)
618
619    #
620    # Miscellaneous utility methods resembling functions in `os`
621    #
622    def getcwd(self):
623        """
624        Return the current path name.
625        """
626        return self._cached_current_dir
627
628    def chdir(self, path):
629        """
630        Change the directory on the host.
631        """
632        path = ftputil.tool.as_str_path(path)
633        with ftputil.error.ftplib_error_to_ftp_os_error:
634            self._session.cwd(path)
635        # The path given as the argument is relative to the old current
636        # directory, therefore join them.
637        self._cached_current_dir = self.path.normpath(
638            self.path.join(self._cached_current_dir, path)
639        )
640
641    # Ignore unused argument `mode`
642    # pylint: disable=unused-argument
643    def mkdir(self, path, mode=None):
644        """
645        Make the directory path on the remote host. The argument `mode` is
646        ignored and only "supported" for similarity with `os.mkdir`.
647        """
648        path = ftputil.tool.as_str_path(path)
649
650        def command(self, path):
651            """Callback function."""
652            with ftputil.error.ftplib_error_to_ftp_os_error:
653                self._session.mkd(path)
654
655        self._robust_ftp_command(command, path)
656
657    # TODO: The virtual directory support doesn't have unit tests yet because
658    # the mocking most likely would be quite complicated. The tests should be
659    # added when mainly the `mock` library is used instead of the mock code in
660    # `test.mock_ftplib`.
661    #
662    # Ignore unused argument `mode`
663    # pylint: disable=unused-argument
664    def makedirs(self, path, mode=None, exist_ok=False):
665        """
666        Make the directory `path`, but also make not yet existing intermediate
667        directories, like `os.makedirs`.
668
669        The value of `mode` is only accepted for compatibility with
670        `os.makedirs` but otherwise ignored.
671
672        If `exist_ok` is `False` (the default) and the leaf directory exists,
673        raise a `PermanentError` with `errno` 17.
674        """
675        path = ftputil.tool.as_str_path(path)
676        path = self.path.abspath(path)
677        directories = path.split(self.sep)
678        old_dir = self.getcwd()
679        try:
680            # Try to build the directory chain from the "uppermost" to the
681            # "lowermost" directory.
682            for index in range(1, len(directories)):
683                # Re-insert the separator which got lost by using `path.split`.
684                next_directory = self.sep + self.path.join(*directories[: index + 1])
685                # If we have "virtual directories" (see #86), just listing the
686                # parent directory won't tell us if a directory actually
687                # exists. So try to change into the directory.
688                try:
689                    self.chdir(next_directory)
690                except ftputil.error.PermanentError:
691                    # Directory presumably doesn't exist.
692                    try:
693                        self.mkdir(next_directory)
694                    except ftputil.error.PermanentError:
695                        # Find out the cause of the error. Re-raise the
696                        # exception only if the directory didn't exist already,
697                        # else something went _really_ wrong, e. g. there's a
698                        # regular file with the name of the directory.
699                        if not self.path.isdir(next_directory):
700                            raise
701                else:
702                    # Directory exists. If we are at the last directory
703                    # component and `exist_ok` is `False`, this is an error.
704                    if (index == len(directories) - 1) and (not exist_ok):
705                        # Before PEP 3151, if `exist_ok` is `False`, trying to
706                        # create an existing directory in the local file system
707                        # results in an `OSError` with `errno.EEXIST, so
708                        # emulate this also for FTP.
709                        ftp_os_error = ftputil.error.PermanentError(
710                            "path {!r} exists".format(path)
711                        )
712                        ftp_os_error.errno = errno.EEXIST
713                        raise ftp_os_error
714        finally:
715            self.chdir(old_dir)
716
717    def rmdir(self, path):
718        """
719        Remove the _empty_ directory `path` on the remote host.
720
721        Compatibility note:
722
723        Previous versions of ftputil could possibly delete non-empty
724        directories as well, - if the server allowed it. This is no longer
725        supported.
726        """
727        path = ftputil.tool.as_str_path(path)
728        path = self.path.abspath(path)
729        if self.listdir(path):
730            raise ftputil.error.PermanentError("directory '{}' not empty".format(path))
731        # XXX: How does `rmd` work with links?
732        def command(self, path):
733            """Callback function."""
734            with ftputil.error.ftplib_error_to_ftp_os_error:
735                self._session.rmd(path)
736
737        self._robust_ftp_command(command, path)
738        self.stat_cache.invalidate(path)
739
740    def remove(self, path):
741        """
742        Remove the file or link given by `path`.
743
744        Raise a `PermanentError` if the path doesn't exist, but maybe raise
745        other exceptions depending on the state of the server (e. g. timeout).
746        """
747        path = ftputil.tool.as_str_path(path)
748        path = self.path.abspath(path)
749        # Though `isfile` includes also links to files, `islink` is needed to
750        # include links to directories.
751        if (
752            self.path.isfile(path)
753            or self.path.islink(path)
754            or not self.path.exists(path)
755        ):
756            # If the path doesn't exist, let the removal command trigger an
757            # exception with a more appropriate error message.
758            def command(self, path):
759                """Callback function."""
760                with ftputil.error.ftplib_error_to_ftp_os_error:
761                    self._session.delete(path)
762
763            self._robust_ftp_command(command, path)
764        else:
765            raise ftputil.error.PermanentError(
766                "remove/unlink can only delete files and links, " "not directories"
767            )
768        self.stat_cache.invalidate(path)
769
770    unlink = remove
771
772    def rmtree(self, path, ignore_errors=False, onerror=None):
773        """
774        Remove the given remote, possibly non-empty, directory tree. The
775        interface of this method is rather complex, in favor of compatibility
776        with `shutil.rmtree`.
777
778        If `ignore_errors` is set to a true value, errors are ignored. If
779        `ignore_errors` is a false value _and_ `onerror` isn't set, all
780        exceptions occurring during the tree iteration and processing are
781        raised. These exceptions are all of type `PermanentError`.
782
783        To distinguish between error situations, pass in a callable for
784        `onerror`. This callable must accept three arguments: `func`, `path`
785        and `exc_info`. `func` is a bound method object, _for example_
786        `your_host_object.listdir`. `path` is the path that was the recent
787        argument of the respective method (`listdir`, `remove`, `rmdir`).
788        `exc_info` is the exception info as it's got from `sys.exc_info`.
789
790        Implementation note: The code is copied from `shutil.rmtree` in
791        Python 2.4 and adapted to ftputil.
792        """
793        path = ftputil.tool.as_str_path(path)
794        # The following code is an adapted version of Python 2.4's
795        # `shutil.rmtree` function.
796        if ignore_errors:
797
798            def new_onerror(*args):
799                """Do nothing."""
800                # pylint: disable=unused-argument
801                pass
802
803        elif onerror is None:
804
805            def new_onerror(*args):
806                """Re-raise exception."""
807                # pylint: disable=misplaced-bare-raise, unused-argument
808                raise
809
810        else:
811            new_onerror = onerror
812        names = []
813        try:
814            names = self.listdir(path)
815        except ftputil.error.PermanentError:
816            new_onerror(self.listdir, path, sys.exc_info())
817        for name in names:
818            full_name = self.path.join(path, name)
819            try:
820                mode = self.lstat(full_name).st_mode
821            except ftputil.error.PermanentError:
822                mode = 0
823            if stat.S_ISDIR(mode):
824                self.rmtree(full_name, ignore_errors, new_onerror)
825            else:
826                try:
827                    self.remove(full_name)
828                except ftputil.error.PermanentError:
829                    new_onerror(self.remove, full_name, sys.exc_info())
830        try:
831            self.rmdir(path)
832        except ftputil.error.FTPOSError:
833            new_onerror(self.rmdir, path, sys.exc_info())
834
835    def rename(self, source, target):
836        """
837        Rename the source on the FTP host to target.
838        """
839        source = ftputil.tool.as_str_path(source)
840        target = ftputil.tool.as_str_path(target)
841        # The following code is in spirit similar to the code in the method
842        # `_robust_ftp_command`, though we do _not_ do _everything_ imaginable.
843        self._check_inaccessible_login_directory()
844        source_head, source_tail = self.path.split(source)
845        target_head, target_tail = self.path.split(target)
846        paths_contain_whitespace = (" " in source_head) or (" " in target_head)
847        if paths_contain_whitespace and source_head == target_head:
848            # Both items are in the same directory.
849            old_dir = self.getcwd()
850            try:
851                self.chdir(source_head)
852                with ftputil.error.ftplib_error_to_ftp_os_error:
853                    self._session.rename(source_tail, target_tail)
854            finally:
855                self.chdir(old_dir)
856        else:
857            # Use straightforward command.
858            with ftputil.error.ftplib_error_to_ftp_os_error:
859                self._session.rename(source, target)
860
861    # XXX: One could argue to put this method into the `_Stat` class, but I
862    # refrained from that because then `_Stat` would have to know about
863    # `FTPHost`'s `_session` attribute and in turn about `_session`'s `dir`
864    # method.
865    def _dir(self, path):
866        """
867        Return a directory listing as made by FTP's `LIST` command as a list of
868        strings.
869        """
870        # Don't use `self.path.isdir` in this method because that would cause a
871        # call of `(l)stat` and thus a call to `_dir`, so we would end up with
872        # an infinite recursion.
873        def _FTPHost_dir_command(self, path):
874            """Callback function."""
875            lines = []
876
877            def callback(line):
878                """Callback function."""
879                lines.append(ftputil.tool.as_str(line))
880
881            with ftputil.error.ftplib_error_to_ftp_os_error:
882                if self.use_list_a_option:
883                    self._session.dir("-a", path, callback)
884                else:
885                    self._session.dir(path, callback)
886            return lines
887
888        lines = self._robust_ftp_command(
889            _FTPHost_dir_command, path, descend_deeply=True
890        )
891        return lines
892
893    # The `listdir`, `lstat` and `stat` methods don't use `_robust_ftp_command`
894    # because they implicitly already use `_dir` which actually uses
895    # `_robust_ftp_command`.
896
897    def listdir(self, path):
898        """
899        Return a list of directories, files etc. in the directory named `path`.
900
901        If the directory listing from the server can't be parsed with any of
902        the available parsers raise a `ParserError`.
903        """
904        original_path = path
905        path = ftputil.tool.as_str_path(path)
906        items = self._stat._listdir(path)
907        return [ftputil.tool.same_string_type_as(original_path, item) for item in items]
908
909    def lstat(self, path, _exception_for_missing_path=True):
910        """
911        Return an object similar to that returned by `os.lstat`.
912
913        If the directory listing from the server can't be parsed with any of
914        the available parsers, raise a `ParserError`. If the directory _can_ be
915        parsed and the `path` is _not_ found, raise a `PermanentError`.
916
917        (`_exception_for_missing_path` is an implementation aid and _not_
918        intended for use by ftputil clients.)
919        """
920        path = ftputil.tool.as_str_path(path)
921        return self._stat._lstat(path, _exception_for_missing_path)
922
923    def stat(self, path, _exception_for_missing_path=True):
924        """
925        Return info from a "stat" call on `path`.
926
927        If the directory containing `path` can't be parsed, raise a
928        `ParserError`. If the directory containing `path` can be parsed but the
929        `path` can't be found, raise a `PermanentError`. Also raise a
930        `PermanentError` if there's an endless (cyclic) chain of symbolic links
931        "behind" the `path`.
932
933        (`_exception_for_missing_path` is an implementation aid and _not_
934        intended for use by ftputil clients.)
935        """
936        path = ftputil.tool.as_str_path(path)
937        return self._stat._stat(path, _exception_for_missing_path)
938
939    def walk(self, top, topdown=True, onerror=None, followlinks=False):
940        """
941        Iterate over directory tree and return a tuple (dirpath, dirnames,
942        filenames) on each iteration, like the `os.walk` function (see
943        https://docs.python.org/library/os.html#os.walk ).
944        """
945        top = ftputil.tool.as_str_path(top)
946        # The following code is copied from `os.walk` in Python 2.4 and adapted
947        # to ftputil.
948        try:
949            names = self.listdir(top)
950        except ftputil.error.FTPOSError as err:
951            if onerror is not None:
952                onerror(err)
953            return
954        dirs, nondirs = [], []
955        for name in names:
956            if self.path.isdir(self.path.join(top, name)):
957                dirs.append(name)
958            else:
959                nondirs.append(name)
960        if topdown:
961            yield top, dirs, nondirs
962        for name in dirs:
963            path = self.path.join(top, name)
964            if followlinks or not self.path.islink(path):
965                yield from self.walk(path, topdown, onerror, followlinks)
966        if not topdown:
967            yield top, dirs, nondirs
968
969    def chmod(self, path, mode):
970        """
971        Change the mode of a remote `path` (a string) to the integer `mode`.
972        This integer uses the same bits as the mode value returned by the
973        `stat` and `lstat` commands.
974
975        If something goes wrong, raise a `TemporaryError` or a
976        `PermanentError`, according to the status code returned by the server.
977        In particular, a non-existent path usually causes a `PermanentError`.
978        """
979        path = ftputil.tool.as_str_path(path)
980        path = self.path.abspath(path)
981
982        def command(self, path):
983            """Callback function."""
984            with ftputil.error.ftplib_error_to_ftp_os_error:
985                self._session.voidcmd("SITE CHMOD 0{0:o} {1}".format(mode, path))
986
987        self._robust_ftp_command(command, path)
988        self.stat_cache.invalidate(path)
989
990    def __getstate__(self):
991        raise TypeError("cannot serialize FTPHost object")
992
993    #
994    # Context manager methods
995    #
996    def __enter__(self):
997        # Return `self`, so it can be accessed as the variable component of the
998        # `with` statement.
999        return self
1000
1001    def __exit__(self, exc_type, exc_val, exc_tb):
1002        # We don't need the `exc_*` arguments here.
1003        # pylint: disable=unused-argument
1004        self.close()
1005        # Be explicit.
1006        return False
Note: See TracBrowser for help on using the repository browser.