source: ftputil/host.py @ 1924:f4b90fa97634

Last change on this file since 1924:f4b90fa97634 was 1924:f4b90fa97634, checked in by Stefan Schwarzer <sschwarzer@…>, 18 months ago
Update `set_time_shift` docstring Emphasize that the client time is UTC. This is consistent with the new calculation in `_Stat.parse_unix_time`. In previous versions of ftputil, the time shift was supposed to be the difference between the local server time (assumed from the timestamp on the server) and the local client time. However, in some circumstances the client time was retrieved in UTC (namely by `os.path.getmtime`), which was inconsistent with other places where the client time was assumed to be a local time. ticket: 134
File size: 39.6 KB
Line 
1# Copyright (C) 2002-2020, Stefan Schwarzer <sschwarzer@sschwarzer.net>
2# and ftputil contributors (see `doc/contributors.txt`)
3# See the file LICENSE for licensing terms.
4
5"""
6`FTPHost` is the central class of the `ftputil` library.
7
8See `__init__.py` for an example.
9"""
10
11import ftplib
12import stat
13import sys
14import time
15
16import ftputil.error
17import ftputil.file
18import ftputil.file_transfer
19import ftputil.path
20import ftputil.stat
21import ftputil.tool
22
23
24__all__ = ["FTPHost"]
25
26
27# The "protected" attributes PyLint talks about aren't intended for
28# clients of the library. `FTPHost` objects need to use some of these
29# library-internal attributes though.
30# pylint: disable=protected-access
31
32
33#####################################################################
34# `FTPHost` class with several methods similar to those of `os`
35
36
37class FTPHost:
38    """FTP host class."""
39
40    # Implementation notes:
41    #
42    # Upon every request of a file (`FTPFile` object) a new FTP
43    # session is created (or reused from a cache), leading to a child
44    # session of the `FTPHost` object from which the file is requested.
45    #
46    # This is needed because opening an `FTPFile` will make the
47    # local session object wait for the completion of the transfer.
48    # In fact, code like this would block indefinitely, if the `RETR`
49    # request would be made on the `_session` of the object host:
50    #
51    #   host = FTPHost(ftp_server, user, password)
52    #   f = host.open("index.html")
53    #   host.getcwd()   # would block!
54    #
55    # On the other hand, the initially constructed host object will
56    # store references to already established `FTPFile` objects and
57    # reuse an associated connection if its associated `FTPFile`
58    # has been closed.
59
60    def __init__(self, *args, **kwargs):
61        """Abstract initialization of `FTPHost` object."""
62        # Store arguments for later operations.
63        self._args = args
64        self._kwargs = kwargs
65        # XXX: Maybe put the following in a `reset` method.
66        # The time shift setting shouldn't be reset though.
67        # Make a session according to these arguments.
68        self._session = self._make_session()
69        # Simulate `os.path`.
70        self.path = ftputil.path._Path(self)
71        # lstat, stat, listdir services.
72        self._stat = ftputil.stat._Stat(self)
73        self.stat_cache = self._stat._lstat_cache
74        self.stat_cache.enable()
75        with ftputil.error.ftplib_error_to_ftp_os_error:
76            self._cached_current_dir = self.path.normpath(
77                ftputil.tool.as_str_path(self._session.pwd())
78            )
79        # Associated `FTPHost` objects for data transfer.
80        self._children = []
81        # This is only set to something else than `None` if this
82        # instance represents an `FTPFile`.
83        self._file = None
84        # Now opened.
85        self.closed = False
86        # Set curdir, pardir etc. for the remote host. RFC 959 states
87        # that this is, strictly speaking, dependent on the server OS
88        # but it seems to work at least with Unix and Windows servers.
89        self.curdir, self.pardir, self.sep = ".", "..", "/"
90        # Set default time shift (used in `upload_if_newer` and
91        # `download_if_newer`).
92        self.set_time_shift(0.0)
93        # Don't use `LIST -a` option by default. If the server doesn't
94        # understand the `-a` option and interprets it as a path, the
95        # results can be surprising. See ticket #110.
96        self.use_list_a_option = False
97
98    def keep_alive(self):
99        """
100        Try to keep the connection alive in order to avoid server timeouts.
101
102        Note that this won't help if the connection has already timed
103        out! In this case, `keep_alive` will raise an `TemporaryError`.
104        (Actually, if you get a server timeout, the error - for a specific
105        connection - will be permanent.)
106        """
107        # Warning: Don't call this method on `FTPHost` instances which
108        # represent file transfers. This may fail in confusing ways.
109        with ftputil.error.ftplib_error_to_ftp_os_error:
110            # Ignore return value.
111            self._session.pwd()
112
113    #
114    # Dealing with child sessions and file-like objects
115    # (rather low-level)
116    #
117    def _make_session(self):
118        """
119        Return a new session object according to the current state of
120        this `FTPHost` instance.
121        """
122        # Don't modify original attributes below.
123        args = self._args[:]
124        kwargs = self._kwargs.copy()
125        # If a session factory has been given on the instantiation of
126        # this `FTPHost` object, use the same factory for this
127        # `FTPHost` object's child sessions.
128        factory = kwargs.pop("session_factory", ftplib.FTP)
129        with ftputil.error.ftplib_error_to_ftp_os_error:
130            session = factory(*args, **kwargs)
131        return session
132
133    def _copy(self):
134        """Return a copy of this `FTPHost` object."""
135        # The copy includes a new session factory return value (aka
136        # session) but doesn't copy the state of `self.getcwd()`.
137        return self.__class__(*self._args, **self._kwargs)
138
139    def _available_child(self):
140        """
141        Return an available (i. e. one whose `_file` object is closed
142        and doesn't have a timed-out server connection) child
143        (`FTPHost` object) from the pool of children or `None` if
144        there aren't any.
145        """
146        # TODO: Currently timed-out child sessions aren't removed and
147        # may collect over time. In very busy or long running
148        # processes, this might slow down an application because the
149        # same stale child sessions have to be processed again and
150        # again.
151        for host in self._children:
152            # Test for timeouts only after testing for a closed file:
153            # - If a file isn't closed, save time; don't bother to access
154            #   the remote server.
155            # - If a file transfer on the child is in progress, requesting
156            #   the directory is an invalid operation because of the way
157            #   the FTP state machine works (see RFC 959).
158            if host._file.closed:
159                try:
160                    host._session.pwd()
161                # Under high load, a 226 status response from a
162                # previous download may arrive too late, so that it's
163                # "seen" in the `pwd` call. For now, skip the
164                # potential child session; it will be considered again
165                # when `_available_child` is called the next time.
166                except ftplib.error_reply:
167                    continue
168                # Timed-out sessions raise `error_temp`.
169                except ftplib.error_temp:
170                    continue
171                # The server may have closed the connection which may
172                # cause `host._session.getline` to raise an `EOFError`
173                # (see ticket #114).
174                except EOFError:
175                    continue
176                # Under high load, there may be a socket read timeout
177                # during the last FTP file `close` (see ticket #112).
178                # Note that a socket timeout is quite different from
179                # an FTP session timeout.
180                except OSError:
181                    continue
182                else:
183                    # Everything's ok; use this `FTPHost` instance.
184                    return host
185        # Be explicit.
186        return None
187
188    def open(
189        self,
190        path,
191        mode="r",
192        buffering=None,
193        encoding=None,
194        errors=None,
195        newline=None,
196        *,
197        rest=None,
198    ):
199        """
200        Return an open file(-like) object which is associated with
201        this `FTPHost` object.
202
203        The arguments `path`, `mode`, `buffering`, `encoding`,
204        `errors` and `newlines` have the same meaning as for `open`.
205        If `rest` is given as an integer,
206
207        - reading will start at the byte (zero-based) `rest`
208        - writing will overwrite the remote file from byte `rest`
209
210        This method tries to reuse a child but will generate a new one
211        if none is available.
212        """
213        # Support the same arguments as `open`.
214        # pylint: disable=too-many-arguments
215        path = ftputil.tool.as_str_path(path)
216        host = self._available_child()
217        if host is None:
218            host = self._copy()
219            self._children.append(host)
220            host._file = ftputil.file.FTPFile(host)
221        basedir = self.getcwd()
222        # Prepare for changing the directory (see whitespace workaround
223        # in method `_dir`).
224        if host.path.isabs(path):
225            effective_path = path
226        else:
227            effective_path = host.path.join(basedir, path)
228        effective_dir, effective_file = host.path.split(effective_path)
229        try:
230            # This will fail if the directory isn't accessible at all.
231            host.chdir(effective_dir)
232        except ftputil.error.PermanentError:
233            # Similarly to a failed `file` in a local file system,
234            # raise an `IOError`, not an `OSError`.
235            raise ftputil.error.FTPIOError(
236                "remote directory '{}' doesn't "
237                "exist or has insufficient access rights".format(effective_dir)
238            )
239        host._file._open(
240            effective_file,
241            mode=mode,
242            buffering=buffering,
243            encoding=encoding,
244            errors=errors,
245            newline=newline,
246            rest=rest,
247        )
248        if "w" in mode:
249            # Invalidate cache entry because size and timestamps will change.
250            self.stat_cache.invalidate(effective_path)
251        return host._file
252
253    def close(self):
254        """Close host connection."""
255        if self.closed:
256            return
257        # Close associated children.
258        for host in self._children:
259            # Children have a `_file` attribute which is an `FTPFile` object.
260            host._file.close()
261            host.close()
262        # Now deal with ourself.
263        try:
264            with ftputil.error.ftplib_error_to_ftp_os_error:
265                self._session.close()
266        finally:
267            # If something went wrong before, the host/session is
268            # probably defunct and subsequent calls to `close` won't
269            # help either, so consider the host/session closed for
270            # practical purposes.
271            self.stat_cache.clear()
272            self._children = []
273            self.closed = True
274
275    #
276    # Setting a custom directory parser
277    #
278    def set_parser(self, parser):
279        """
280        Set the parser for extracting stat results from directory
281        listings.
282
283        The parser interface is described in the documentation, but
284        here are the most important things:
285
286        - A parser should derive from `ftputil.stat.Parser`.
287
288        - The parser has to implement two methods, `parse_line` and
289          `ignores_line`. For the latter, there's a probably useful
290          default in the class `ftputil.stat.Parser`.
291
292        - `parse_line` should try to parse a line of a directory
293          listing and return a `ftputil.stat.StatResult` instance. If
294          parsing isn't possible, raise `ftputil.error.ParserError`
295          with a useful error message.
296
297        - `ignores_line` should return a true value if the line isn't
298          assumed to contain stat information.
299        """
300        # The cache contents, if any, probably aren't useful.
301        self.stat_cache.clear()
302        # Set the parser explicitly, don't allow "smart" switching anymore.
303        self._stat._parser = parser
304        self._stat._allow_parser_switching = False
305
306    #
307    # Time shift adjustment between client (i. e. us) and server
308    #
309    def set_time_shift(self, time_shift):
310        """
311        Set the time shift value (i. e. the time difference between
312        client and server, where the client time is in UTC) for this
313        `FTPHost` object. By (my) definition, the time shift value is
314        positive if the local time of the server is greater than the
315        local time of the client (for the same physical time), i. e.
316
317            time_shift =def= t_server - t_client_utc
318        <=> t_server = t_client_utc + time_shift
319        <=> t_client_utc = t_server - time_shift
320
321        The time shift is measured in seconds.
322        """
323        # Implicitly set via `set_time_shift` call in constructor
324        # pylint: disable=attribute-defined-outside-init
325        self._time_shift = time_shift
326
327    def time_shift(self):
328        """
329        Return the time shift between FTP server and client. See the
330        docstring of `set_time_shift` for more on this value.
331        """
332        return self._time_shift
333
334    @staticmethod
335    def __rounded_time_shift(time_shift):
336        """
337        Return the given time shift in seconds, but rounded to
338        15-minute units. The argument is also assumed to be given in
339        seconds.
340        """
341        minute = 60.0
342        # Avoid division by zero below.
343        if time_shift == 0:
344            return 0.0
345        # Use a positive value for rounding.
346        absolute_time_shift = abs(time_shift)
347        signum = time_shift / absolute_time_shift
348        # Round absolute time shift to 15-minute units.
349        absolute_rounded_time_shift = int(
350            (absolute_time_shift + (7.5 * minute)) / (15.0 * minute)
351        ) * (15.0 * minute)
352        # Return with correct sign.
353        return signum * absolute_rounded_time_shift
354
355    def __assert_valid_time_shift(self, time_shift):
356        """
357        Perform sanity checks on the time shift value (given in
358        seconds). If the value is invalid, raise a `TimeShiftError`,
359        else simply return `None`.
360        """
361        minute = 60.0  # seconds
362        hour = 60.0 * minute
363        absolute_rounded_time_shift = abs(self.__rounded_time_shift(time_shift))
364        # Test 1: Fail if the absolute time shift is greater than
365        #         a full day (24 hours).
366        if absolute_rounded_time_shift > 24 * hour:
367            raise ftputil.error.TimeShiftError(
368                "time shift abs({0:.2f} s) > 1 day".format(time_shift)
369            )
370        # Test 2: Fail if the deviation between given time shift and
371        #         15-minute units is greater than a certain limit.
372        maximum_deviation = 5 * minute
373        if abs(time_shift - self.__rounded_time_shift(time_shift)) > maximum_deviation:
374            raise ftputil.error.TimeShiftError(
375                "time shift ({0:.2f} s) deviates more than {1:d} s "
376                "from 15-minute units".format(time_shift, int(maximum_deviation))
377            )
378
379    def synchronize_times(self):
380        """
381        Synchronize the local times of FTP client and server. This
382        is necessary to let `upload_if_newer` and `download_if_newer`
383        work correctly. If `synchronize_times` isn't applicable
384        (see below), the time shift can still be set explicitly with
385        `set_time_shift`.
386
387        This implementation of `synchronize_times` requires _all_ of
388        the following:
389
390        - The connection between server and client is established.
391        - The client has write access to the directory that is
392          current when `synchronize_times` is called.
393
394        The common usage pattern of `synchronize_times` is to call it
395        directly after the connection is established. (As can be
396        concluded from the points above, this requires write access
397        to the login directory.)
398
399        If `synchronize_times` fails, it raises a `TimeShiftError`.
400        """
401        helper_file_name = "_ftputil_sync_"
402        # Open a dummy file for writing in the current directory
403        # on the FTP host, then close it.
404        try:
405            # May raise `FTPIOError` if directory isn't writable.
406            file_ = self.open(helper_file_name, "w")
407            file_.close()
408        except ftputil.error.FTPIOError:
409            raise ftputil.error.TimeShiftError(
410                """couldn't write helper file in directory '{}'""".format(self.getcwd())
411            )
412        # If everything worked up to here it should be possible to stat
413        # and then remove the just-written file.
414        try:
415            server_time = self.path.getmtime(helper_file_name)
416            self.unlink(helper_file_name)
417        except ftputil.error.FTPOSError:
418            # If we got a `TimeShiftError` exception above, we
419            # should't come here: if we didn't get a `TimeShiftError`
420            # above, deletion should be possible. The only reason for
421            # an exception I can think of here is a race condition by
422            # removing the helper file or write permission from the
423            # directory or helper file after it has been written to.
424            raise ftputil.error.TimeShiftError(
425                "could write helper file but not unlink it"
426            )
427        # Calculate the difference between server and client.
428        now = time.time()
429        time_shift = server_time - now
430        # As the time shift for this host instance isn't set yet, the
431        # directory parser will calculate times one year in the past if
432        # the time zone of the server is east from ours. Thus the time
433        # shift will be off by a year as well (see ticket #55).
434        if time_shift < -360 * 24 * 60 * 60:
435            # Re-add one year and re-calculate the time shift. We don't
436            # know how many days made up that year (it might have been
437            # a leap year), so go the route via `time.localtime` and
438            # `time.mktime`.
439            server_time_struct = time.localtime(server_time)
440            server_time_struct = (server_time_struct.tm_year + 1,) + server_time_struct[
441                1:
442            ]
443            server_time = time.mktime(server_time_struct)
444            time_shift = server_time - now
445        # Do some sanity checks.
446        self.__assert_valid_time_shift(time_shift)
447        # If tests passed, store the time difference as time shift value.
448        self.set_time_shift(self.__rounded_time_shift(time_shift))
449
450    #
451    # Operations based on file-like objects (rather high-level),
452    # like upload and download
453    #
454    # XXX: This has a different API from `shutil.copyfileobj`, on which this
455    # method is modeled. But I don't think it makes sense to change this method
456    # here because the method is probably rarely used and a change would break
457    # client code.
458    @staticmethod
459    def copyfileobj(
460        source,
461        target,
462        max_chunk_size=ftputil.file_transfer.MAX_COPY_CHUNK_SIZE,
463        callback=None,
464    ):
465        """
466        Copy data from file-like object `source` to file-like object
467        `target`.
468        """
469        ftputil.file_transfer.copyfileobj(source, target, max_chunk_size, callback)
470
471    def _upload_files(self, source_path, target_path):
472        """
473        Return a `LocalFile` and `RemoteFile` as source and target,
474        respectively.
475
476        The strings `source_path` and `target_path` are the (absolute
477        or relative) paths of the local and the remote file, respectively.
478        """
479        source_file = ftputil.file_transfer.LocalFile(source_path, "rb")
480        # Passing `self` (the `FTPHost` instance) here is correct.
481        target_file = ftputil.file_transfer.RemoteFile(self, target_path, "wb")
482        return source_file, target_file
483
484    def upload(self, source, target, callback=None):
485        """
486        Upload a file from the local source (name) to the remote
487        target (name).
488
489        If a callable `callback` is given, it's called after every
490        chunk of transferred data. The chunk size is a constant
491        defined in `file_transfer`. The callback will be called with a
492        single argument, the data chunk that was transferred before
493        the callback was called.
494        """
495        target = ftputil.tool.as_str_path(target)
496        source_file, target_file = self._upload_files(source, target)
497        ftputil.file_transfer.copy_file(
498            source_file, target_file, conditional=False, callback=callback
499        )
500
501    def upload_if_newer(self, source, target, callback=None):
502        """
503        Upload a file only if it's newer than the target on the
504        remote host or if the target file does not exist. See the
505        method `upload` for the meaning of the parameters.
506
507        If an upload was necessary, return `True`, else return `False`.
508
509        If a callable `callback` is given, it's called after every
510        chunk of transferred data. The chunk size is a constant
511        defined in `file_transfer`. The callback will be called with a
512        single argument, the data chunk that was transferred before
513        the callback was called.
514        """
515        target = ftputil.tool.as_str_path(target)
516        source_file, target_file = self._upload_files(source, target)
517        return ftputil.file_transfer.copy_file(
518            source_file, target_file, conditional=True, callback=callback
519        )
520
521    def _download_files(self, source_path, target_path):
522        """
523        Return a `RemoteFile` and `LocalFile` as source and target,
524        respectively.
525
526        The strings `source_path` and `target_path` are the (absolute
527        or relative) paths of the remote and the local file, respectively.
528        """
529        source_file = ftputil.file_transfer.RemoteFile(self, source_path, "rb")
530        target_file = ftputil.file_transfer.LocalFile(target_path, "wb")
531        return source_file, target_file
532
533    def download(self, source, target, callback=None):
534        """
535        Download a file from the remote source (name) to the local
536        target (name).
537
538        If a callable `callback` is given, it's called after every
539        chunk of transferred data. The chunk size is a constant
540        defined in `file_transfer`. The callback will be called with a
541        single argument, the data chunk that was transferred before
542        the callback was called.
543        """
544        source = ftputil.tool.as_str_path(source)
545        source_file, target_file = self._download_files(source, target)
546        ftputil.file_transfer.copy_file(
547            source_file, target_file, conditional=False, callback=callback
548        )
549
550    def download_if_newer(self, source, target, callback=None):
551        """
552        Download a file only if it's newer than the target on the
553        local host or if the target file does not exist. See the
554        method `download` for the meaning of the parameters.
555
556        If a download was necessary, return `True`, else return
557        `False`.
558
559        If a callable `callback` is given, it's called after every
560        chunk of transferred data. The chunk size is a constant
561        defined in `file_transfer`. The callback will be called with a
562        single argument, the data chunk that was transferred before
563        the callback was called.
564        """
565        source = ftputil.tool.as_str_path(source)
566        source_file, target_file = self._download_files(source, target)
567        return ftputil.file_transfer.copy_file(
568            source_file, target_file, conditional=True, callback=callback
569        )
570
571    #
572    # Helper methods to descend into a directory before executing a command
573    #
574    def _check_inaccessible_login_directory(self):
575        """
576        Raise an `InaccessibleLoginDirError` exception if we can't
577        change to the login directory. This test is only reliable if
578        the current directory is the login directory.
579        """
580        presumable_login_dir = self.getcwd()
581        # Bail out with an internal error rather than modify the
582        # current directory without hope of restoration.
583        try:
584            self.chdir(presumable_login_dir)
585        except ftputil.error.PermanentError:
586            raise ftputil.error.InaccessibleLoginDirError(
587                "directory '{}' is not accessible".format(presumable_login_dir)
588            )
589
590    def _robust_ftp_command(self, command, path, descend_deeply=False):
591        """
592        Run an FTP command on a path. The return value of the method
593        is the return value of the command.
594
595        If `descend_deeply` is true (the default is false), descend
596        deeply, i. e. change the directory to the end of the path.
597        """
598        # If we can't change to the yet-current directory, the code
599        # below won't work (see below), so in this case rather raise
600        # an exception than giving wrong results.
601        self._check_inaccessible_login_directory()
602        # Some FTP servers don't behave as expected if the directory
603        # portion of the path contains whitespace; some even yield
604        # strange results if the command isn't executed in the
605        # current directory. Therefore, change to the directory
606        # which contains the item to run the command on and invoke
607        # the command just there.
608        #
609        # Remember old working directory.
610        old_dir = self.getcwd()
611        try:
612            if descend_deeply:
613                # Invoke the command in (not: on) the deepest directory.
614                self.chdir(path)
615                # Workaround for some servers that give recursive
616                # listings when called with a dot as path; see issue #33,
617                # http://ftputil.sschwarzer.net/trac/ticket/33
618                return command(self, "")
619            else:
620                # Invoke the command in the "next to last" directory.
621                head, tail = self.path.split(path)
622                self.chdir(head)
623                return command(self, tail)
624        finally:
625            self.chdir(old_dir)
626
627    #
628    # Miscellaneous utility methods resembling functions in `os`
629    #
630    def getcwd(self):
631        """Return the current path name."""
632        return self._cached_current_dir
633
634    def chdir(self, path):
635        """Change the directory on the host."""
636        path = ftputil.tool.as_str_path(path)
637        with ftputil.error.ftplib_error_to_ftp_os_error:
638            self._session.cwd(path)
639        # The path given as the argument is relative to the old current
640        # directory, therefore join them.
641        self._cached_current_dir = self.path.normpath(
642            self.path.join(self._cached_current_dir, path)
643        )
644
645    # Ignore unused argument `mode`
646    # pylint: disable=unused-argument
647    def mkdir(self, path, mode=None):
648        """
649        Make the directory path on the remote host. The argument
650        `mode` is ignored and only "supported" for similarity with
651        `os.mkdir`.
652        """
653        path = ftputil.tool.as_str_path(path)
654
655        def command(self, path):
656            """Callback function."""
657            with ftputil.error.ftplib_error_to_ftp_os_error:
658                self._session.mkd(path)
659
660        self._robust_ftp_command(command, path)
661
662    # TODO: The virtual directory support doesn't have unit tests yet
663    # because the mocking most likely would be quite complicated. The
664    # tests should be added when mainly the `mock` library is used
665    # instead of the mock code in `test.mock_ftplib`.
666    #
667    # Ignore unused argument `mode`
668    # pylint: disable=unused-argument
669    def makedirs(self, path, mode=None):
670        """
671        Make the directory `path`, but also make not yet existing
672        intermediate directories, like `os.makedirs`. The value of
673        `mode` is only accepted for compatibility with `os.makedirs`
674        but otherwise ignored.
675        """
676        path = ftputil.tool.as_str_path(path)
677        path = self.path.abspath(path)
678        directories = path.split(self.sep)
679        old_dir = self.getcwd()
680        try:
681            # Try to build the directory chain from the "uppermost" to
682            # the "lowermost" directory.
683            for index in range(1, len(directories)):
684                # Re-insert the separator which got lost by using
685                # `path.split`.
686                next_directory = self.sep + self.path.join(*directories[: index + 1])
687                # If we have "virtual directories" (see #86), just
688                # listing the parent directory won't tell us if a
689                # directory actually exists. So try to change into the
690                # directory.
691                try:
692                    self.chdir(next_directory)
693                except ftputil.error.PermanentError:
694                    try:
695                        self.mkdir(next_directory)
696                    except ftputil.error.PermanentError:
697                        # Find out the cause of the error. Re-raise
698                        # the exception only if the directory didn't
699                        # exist already, else something went _really_
700                        # wrong, e. g. there's a regular file with the
701                        # name of the directory.
702                        if not self.path.isdir(next_directory):
703                            raise
704        finally:
705            self.chdir(old_dir)
706
707    def rmdir(self, path):
708        """
709        Remove the _empty_ directory `path` on the remote host.
710
711        Compatibility note:
712
713        Previous versions of ftputil could possibly delete non-
714        empty directories as well, - if the server allowed it. This
715        is no longer supported.
716        """
717        path = ftputil.tool.as_str_path(path)
718        path = self.path.abspath(path)
719        if self.listdir(path):
720            raise ftputil.error.PermanentError("directory '{}' not empty".format(path))
721        # XXX: How does `rmd` work with links?
722        def command(self, path):
723            """Callback function."""
724            with ftputil.error.ftplib_error_to_ftp_os_error:
725                self._session.rmd(path)
726
727        self._robust_ftp_command(command, path)
728        self.stat_cache.invalidate(path)
729
730    def remove(self, path):
731        """
732        Remove the file or link given by `path`.
733
734        Raise a `PermanentError` if the path doesn't exist, but maybe
735        raise other exceptions depending on the state of the server
736        (e. g. timeout).
737        """
738        path = ftputil.tool.as_str_path(path)
739        path = self.path.abspath(path)
740        # Though `isfile` includes also links to files, `islink`
741        # is needed to include links to directories.
742        if (
743            self.path.isfile(path)
744            or self.path.islink(path)
745            or not self.path.exists(path)
746        ):
747            # If the path doesn't exist, let the removal command trigger
748            # an exception with a more appropriate error message.
749            def command(self, path):
750                """Callback function."""
751                with ftputil.error.ftplib_error_to_ftp_os_error:
752                    self._session.delete(path)
753
754            self._robust_ftp_command(command, path)
755        else:
756            raise ftputil.error.PermanentError(
757                "remove/unlink can only delete files and links, " "not directories"
758            )
759        self.stat_cache.invalidate(path)
760
761    unlink = remove
762
763    def rmtree(self, path, ignore_errors=False, onerror=None):
764        """
765        Remove the given remote, possibly non-empty, directory tree.
766        The interface of this method is rather complex, in favor of
767        compatibility with `shutil.rmtree`.
768
769        If `ignore_errors` is set to a true value, errors are ignored.
770        If `ignore_errors` is a false value _and_ `onerror` isn't set,
771        all exceptions occurring during the tree iteration and
772        processing are raised. These exceptions are all of type
773        `PermanentError`.
774
775        To distinguish between error situations, pass in a callable
776        for `onerror`. This callable must accept three arguments:
777        `func`, `path` and `exc_info`. `func` is a bound method
778        object, _for example_ `your_host_object.listdir`. `path` is
779        the path that was the recent argument of the respective method
780        (`listdir`, `remove`, `rmdir`). `exc_info` is the exception
781        info as it's got from `sys.exc_info`.
782
783        Implementation note: The code is copied from `shutil.rmtree`
784        in Python 2.4 and adapted to ftputil.
785        """
786        path = ftputil.tool.as_str_path(path)
787        # The following code is an adapted version of Python 2.4's
788        # `shutil.rmtree` function.
789        if ignore_errors:
790
791            def new_onerror(*args):
792                """Do nothing."""
793                # pylint: disable=unused-argument
794                pass
795
796        elif onerror is None:
797
798            def new_onerror(*args):
799                """Re-raise exception."""
800                # pylint: disable=misplaced-bare-raise, unused-argument
801                raise
802
803        else:
804            new_onerror = onerror
805        names = []
806        try:
807            names = self.listdir(path)
808        except ftputil.error.PermanentError:
809            new_onerror(self.listdir, path, sys.exc_info())
810        for name in names:
811            full_name = self.path.join(path, name)
812            try:
813                mode = self.lstat(full_name).st_mode
814            except ftputil.error.PermanentError:
815                mode = 0
816            if stat.S_ISDIR(mode):
817                self.rmtree(full_name, ignore_errors, new_onerror)
818            else:
819                try:
820                    self.remove(full_name)
821                except ftputil.error.PermanentError:
822                    new_onerror(self.remove, full_name, sys.exc_info())
823        try:
824            self.rmdir(path)
825        except ftputil.error.FTPOSError:
826            new_onerror(self.rmdir, path, sys.exc_info())
827
828    def rename(self, source, target):
829        """Rename the source on the FTP host to target."""
830        source = ftputil.tool.as_str_path(source)
831        target = ftputil.tool.as_str_path(target)
832        # The following code is in spirit similar to the code in the
833        # method `_robust_ftp_command`, though we do _not_ do
834        # _everything_ imaginable.
835        self._check_inaccessible_login_directory()
836        source_head, source_tail = self.path.split(source)
837        target_head, target_tail = self.path.split(target)
838        paths_contain_whitespace = (" " in source_head) or (" " in target_head)
839        if paths_contain_whitespace and source_head == target_head:
840            # Both items are in the same directory.
841            old_dir = self.getcwd()
842            try:
843                self.chdir(source_head)
844                with ftputil.error.ftplib_error_to_ftp_os_error:
845                    self._session.rename(source_tail, target_tail)
846            finally:
847                self.chdir(old_dir)
848        else:
849            # Use straightforward command.
850            with ftputil.error.ftplib_error_to_ftp_os_error:
851                self._session.rename(source, target)
852
853    # XXX: One could argue to put this method into the `_Stat` class,
854    # but I refrained from that because then `_Stat` would have to
855    # know about `FTPHost`'s `_session` attribute and in turn about
856    # `_session`'s `dir` method.
857    def _dir(self, path):
858        """
859        Return a directory listing as made by FTP's `LIST` command as
860        a list of strings.
861        """
862        # Don't use `self.path.isdir` in this method because that
863        # would cause a call of `(l)stat` and thus a call to `_dir`,
864        # so we would end up with an infinite recursion.
865        def _FTPHost_dir_command(self, path):
866            """Callback function."""
867            lines = []
868
869            def callback(line):
870                """Callback function."""
871                lines.append(ftputil.tool.as_str(line))
872
873            with ftputil.error.ftplib_error_to_ftp_os_error:
874                if self.use_list_a_option:
875                    self._session.dir("-a", path, callback)
876                else:
877                    self._session.dir(path, callback)
878            return lines
879
880        lines = self._robust_ftp_command(
881            _FTPHost_dir_command, path, descend_deeply=True
882        )
883        return lines
884
885    # The `listdir`, `lstat` and `stat` methods don't use
886    # `_robust_ftp_command` because they implicitly already use
887    # `_dir` which actually uses `_robust_ftp_command`.
888
889    def listdir(self, path):
890        """
891        Return a list of directories, files etc. in the directory
892        named `path`.
893
894        If the directory listing from the server can't be parsed with
895        any of the available parsers raise a `ParserError`.
896        """
897        original_path = path
898        path = ftputil.tool.as_str_path(path)
899        items = self._stat._listdir(path)
900        return [ftputil.tool.same_string_type_as(original_path, item) for item in items]
901
902    def lstat(self, path, _exception_for_missing_path=True):
903        """
904        Return an object similar to that returned by `os.lstat`.
905
906        If the directory listing from the server can't be parsed with
907        any of the available parsers, raise a `ParserError`. If the
908        directory _can_ be parsed and the `path` is _not_ found, raise
909        a `PermanentError`.
910
911        (`_exception_for_missing_path` is an implementation aid and
912        _not_ intended for use by ftputil clients.)
913        """
914        path = ftputil.tool.as_str_path(path)
915        return self._stat._lstat(path, _exception_for_missing_path)
916
917    def stat(self, path, _exception_for_missing_path=True):
918        """
919        Return info from a "stat" call on `path`.
920
921        If the directory containing `path` can't be parsed, raise a
922        `ParserError`. If the directory containing `path` can be
923        parsed but the `path` can't be found, raise a
924        `PermanentError`. Also raise a `PermanentError` if there's an
925        endless (cyclic) chain of symbolic links "behind" the `path`.
926
927        (`_exception_for_missing_path` is an implementation aid and
928        _not_ intended for use by ftputil clients.)
929        """
930        path = ftputil.tool.as_str_path(path)
931        return self._stat._stat(path, _exception_for_missing_path)
932
933    def walk(self, top, topdown=True, onerror=None, followlinks=False):
934        """
935        Iterate over directory tree and return a tuple (dirpath,
936        dirnames, filenames) on each iteration, like the `os.walk`
937        function (see https://docs.python.org/library/os.html#os.walk ).
938        """
939        top = ftputil.tool.as_str_path(top)
940        # The following code is copied from `os.walk` in Python 2.4
941        # and adapted to ftputil.
942        try:
943            names = self.listdir(top)
944        except ftputil.error.FTPOSError as err:
945            if onerror is not None:
946                onerror(err)
947            return
948        dirs, nondirs = [], []
949        for name in names:
950            if self.path.isdir(self.path.join(top, name)):
951                dirs.append(name)
952            else:
953                nondirs.append(name)
954        if topdown:
955            yield top, dirs, nondirs
956        for name in dirs:
957            path = self.path.join(top, name)
958            if followlinks or not self.path.islink(path):
959                yield from self.walk(path, topdown, onerror, followlinks)
960        if not topdown:
961            yield top, dirs, nondirs
962
963    def chmod(self, path, mode):
964        """
965        Change the mode of a remote `path` (a string) to the integer
966        `mode`. This integer uses the same bits as the mode value
967        returned by the `stat` and `lstat` commands.
968
969        If something goes wrong, raise a `TemporaryError` or a
970        `PermanentError`, according to the status code returned by
971        the server. In particular, a non-existent path usually
972        causes a `PermanentError`.
973        """
974        path = ftputil.tool.as_str_path(path)
975        path = self.path.abspath(path)
976
977        def command(self, path):
978            """Callback function."""
979            with ftputil.error.ftplib_error_to_ftp_os_error:
980                self._session.voidcmd("SITE CHMOD 0{0:o} {1}".format(mode, path))
981
982        self._robust_ftp_command(command, path)
983        self.stat_cache.invalidate(path)
984
985    def __getstate__(self):
986        raise TypeError("cannot serialize FTPHost object")
987
988    #
989    # Context manager methods
990    #
991    def __enter__(self):
992        # Return `self`, so it can be accessed as the variable
993        # component of the `with` statement.
994        return self
995
996    def __exit__(self, exc_type, exc_val, exc_tb):
997        # We don't need the `exc_*` arguments here.
998        # pylint: disable=unused-argument
999        self.close()
1000        # Be explicit.
1001        return False
Note: See TracBrowser for help on using the repository browser.