source: ftputil/file.py @ 1334:bd741c9b6153

Last change on this file since 1334:bd741c9b6153 was 1334:bd741c9b6153, checked in by Stefan Schwarzer <sschwarzer@…>, 7 years ago
Use a slightly dump but effective way to give a `socket.makefile` file interface of a `BufferedReader` or `BufferedWriter` under Python 2. The previous approach, still visible in `_FTPFile._wrapped_file`, still didn't work properly after a lot of fiddling with `__getattr__` and `__setattr__`.
File size: 11.2 KB
Line 
1# Copyright (C) 2003-2013, Stefan Schwarzer <sschwarzer@sschwarzer.net>
2# Copyright (C) 2008, Roger Demetrescu <roger.demetrescu@gmail.com>
3# See the file LICENSE for licensing terms.
4
5"""
6ftputil.file - support for file-like objects on FTP servers
7"""
8
9from __future__ import print_function
10from __future__ import unicode_literals
11
12import io
13
14import ftputil.compat
15import ftputil.error
16
17
18# This module shouldn't be used by clients of the ftputil library.
19__all__ = []
20
21
22class BufferedReaderWriter(io.BufferedIOBase):
23    """
24    Adapt a file object returned from `socket.makefile` to the
25    interfaces of `io.BufferedReader` or `io.BufferedWriter`, so that
26    the new object can be wrapped by `io.TextIOWrapper`.
27
28    This is only needed with Python 2, since in Python 3
29    `socket.makefile` already returns a `BufferedReader` or
30    `BufferedWriter` object (depending on mode).
31    """
32
33    def __init__(self, fobj, is_readable=False, is_writable=False):
34        # This is the return value of `socket.makefile` and is already
35        # buffered.
36        self.raw = fobj
37        self._is_readable = is_readable
38        self._is_writable = is_writable
39
40    @property
41    def closed(self):
42        return self.raw.closed
43
44    def close(self):
45        self.raw.close()
46
47    def fileno(self):
48        return self.raw.fileno()
49
50    def isatty(self):
51        # It's highly unlikely that this is interactive.
52        return False
53
54    def seekable(self):
55        return False
56
57    #
58    # Interface for `BufferedReader`
59    #
60    def readable(self):
61        return self._is_readable
62
63    def read(self, *arg):
64        return self.raw.read(*arg)
65
66    read1 = read
67
68    def readline(self, *arg):
69        return self.raw.readline(*arg)
70
71    def readlines(self, *arg):
72        return self.raw.readlines(*arg)
73
74    def readinto(self, bytearray_):
75        data = self.raw.read(len(bytearray_))
76        bytearray_[:len(data)] = data
77        return len(data)
78
79    #
80    # Interface for `BufferedWriter`
81    #
82    def writable(self):
83        return self._is_writable
84
85    def flush(self):
86        self.raw.flush()
87
88    def write(self, bytes_or_bytearray):
89        # `BufferedWriter.write` has to return the number of written
90        # bytes. Since we don't really know how many bytes got
91        # actually written, return the length of the full data, but
92        # also call `flush` to increase the chance that all bytes are
93        # written.
94        self.raw.write(bytes_or_bytearray)
95        # TODO: Measure impact of flushing for many small writes.
96        self.flush()
97        return len(bytes_or_bytearray)
98
99    def writelines(self, lines):
100        self.raw.writelines(lines)
101
102
103class _FTPFile(object):
104    """
105    Represents a file-like object associated with an FTP host. File
106    and socket are closed appropriately if the `close` method is
107    called.
108    """
109
110    # Set timeout in seconds when closing file connections (see ticket #51).
111    _close_timeout = 5
112
113    def __init__(self, host):
114        """Construct the file(-like) object."""
115        self._host = host
116        self._session = host._session
117        # The file is still closed.
118        self.closed = True
119        self._conn = None
120        self._fobj = None
121
122    def _wrapped_file(self, fobj, is_readable=False, is_writable=False):
123        """
124        Return a new file-like object which wraps `fobj` and in
125        addition has the `readable`, `readinto` and `writable` methods
126        that `BufferedReader` or `BufferedWriter` require.
127        """
128        # I tried to assign the missing methods as bound methods
129        # directly to `fobj`, but this seemingly isn't possible with
130        # the file object returned from `socket.makefile`.
131        class Wrapper(io.RawIOBase):
132            def __init__(self, fobj):
133                super(Wrapper, self).__setattr__("_fobj", fobj)
134            def readable(self):
135                return is_readable
136            def writable(self):
137                return is_writable
138            def readinto(self, bytearray_):
139                data = self._fobj.read(len(bytearray_))
140                bytearray_[:len(data)] = data
141                return len(data)
142            def write(self, bytes_):
143                # `BufferedWriter` expects the number of written bytes
144                # as return value. Since we don't really know how many
145                # bytes are written, return the length of the full
146                # data, but also call `flush` to increase the chance
147                # that actually all bytes are written.
148                print("=== type(bytes):", type(bytes_))
149                # Use slice in case we get a `memoryview` object.
150                self._fobj.write(bytes_[:])
151                self._fobj.flush()
152                return len(bytes_)
153            def __getattr__(self, name):
154                if name == "__IOBase_closed":
155                    result = super(Wrapper, self).__getattr__(name)
156                    return result
157                else:
158                    result = getattr(self._fobj, name)
159                    return result
160            def __setattr__(self, name, value):
161                if name == "__IOBase_closed":
162                    # Delegate to this (`RawIOBase`) instance.
163                    return super(Wrapper, self).__setattr__(name, value)
164                else:
165                    return setattr(self._fobj, name, value)
166        return Wrapper(fobj)
167
168    def _open(self, path, mode, buffering=None, encoding=None, errors=None,
169              newline=None):
170        """
171        Open the remote file with given path name and mode.
172
173        Contrary to the `open` builtin, this method returns `None`,
174        instead this file object is modified in-place.
175        """
176        # Python 3.x's `socket.makefile` supports the same interface
177        # as the new `open` builtin, but Python 2.x supports a mode,
178        # but neither buffering nor encoding/decoding. Therefore, to
179        # make the code work on Python 2.x _and_ 3.x, create an
180        # unbuffered binary file and possibly wrap it.
181        #
182        # Check mode.
183        if "a" in mode:
184            raise ftputil.error.FTPIOError("append mode not supported")
185        if mode not in ("r", "rb", "rt", "w", "wb", "wt"):
186            raise ftputil.error.FTPIOError("invalid mode '{0}'".format(mode))
187        if "b" in mode and "t" in mode:
188            # Raise a `ValueError` like Python would.
189            raise ValueError("can't have text and binary mode at once")
190        # Convenience variables
191        is_bin_mode = "b" in mode
192        is_read_mode = "r" in mode
193        # Always use binary mode (see above).
194        transfer_type = "I"
195        command = 'TYPE {0}'.format(transfer_type)
196        with ftputil.error.ftplib_error_to_ftp_io_error:
197            self._session.voidcmd(command)
198        # Make transfer command.
199        command_type = ('STOR', 'RETR')[is_read_mode]
200        command = '{0} {1}'.format(command_type, path)
201        # Force to binary regardless of transfer type (see above).
202        makefile_mode = mode
203        makefile_mode = makefile_mode.replace("t", "")
204        if not "b" in makefile_mode:
205            makefile_mode += "b"
206        # Get connection and file object.
207        with ftputil.error.ftplib_error_to_ftp_io_error:
208            self._conn = self._session.transfercmd(command)
209        # The actual file object. Under Python 3, this will already
210        # be wrapped by a `BufferedReader` or `BufferedWriter`.
211        fobj = self._conn.makefile(makefile_mode)
212        if ftputil.compat.python_version == 2:
213            if is_read_mode:
214                fobj = BufferedReaderWriter(fobj, is_readable=True)
215            else:
216                fobj = BufferedReaderWriter(fobj, is_writable=True)
217        if not is_bin_mode:
218            fobj = io.TextIOWrapper(fobj, encoding=encoding,
219                                    errors=errors, newline=newline)
220        self._fobj = fobj
221        # This comes last so that `close` won't try to close `_FTPFile`
222        # objects without `_conn` and `_fo` attributes in case of an error.
223        self.closed = False
224
225    def __iter__(self):
226        """Return a file iterator."""
227        return self
228
229    def __next__(self):
230        """
231        Return the next line or raise `StopIteration`, if there are
232        no more.
233        """
234        # Apply implicit line ending conversion.
235        line = self.readline()
236        if line:
237            return line
238        else:
239            raise StopIteration
240
241    # Although Python 2.6+ has the `next` builtin function already, it
242    # still requires iterators to have a `next` method.
243    next = __next__
244
245    #
246    # Context manager methods
247    #
248    def __enter__(self):
249        # Return `self`, so it can be accessed as the variable
250        # component of the `with` statement.
251        return self
252
253    def __exit__(self, exc_type, exc_val, exc_tb):
254        # We don't need the `exc_*` arguments here
255        # pylint: disable=W0613
256        self.close()
257        # Be explicit
258        return False
259
260    #
261    # Other attributes
262    #
263    def __getattr__(self, attr_name):
264        """
265        Handle requests for attributes unknown to `_FTPFile` objects:
266        delegate the requests to the contained file object.
267        """
268        if attr_name in ("encoding flush isatty fileno read readline "
269                         "readlines seek tell truncate name softspace "
270                         "write writelines".split()):
271            return getattr(self._fobj, attr_name)
272        raise AttributeError(
273              "'FTPFile' object has no attribute '{0}'".format(attr_name))
274
275    # TODO: Implement `__dir__`? (See
276    # http://docs.python.org/py3k/whatsnew/2.6.html#other-language-changes )
277
278    def close(self):
279        """Close the `FTPFile`."""
280        if self.closed:
281            return
282        # Timeout value to restore, see below.
283        # Statement works only before the try/finally statement,
284        # otherwise Python raises an `UnboundLocalError`.
285        old_timeout = self._session.sock.gettimeout()
286        try:
287            self._fobj.close()
288            self._fobj = None
289            with ftputil.error.ftplib_error_to_ftp_io_error:
290                self._conn.close()
291            # Set a timeout to prevent waiting until server timeout
292            # if we have a server blocking here like in ticket #51.
293            self._session.sock.settimeout(self._close_timeout)
294            try:
295                with ftputil.error.ftplib_error_to_ftp_io_error:
296                    self._session.voidresp()
297            except ftputil.error.FTPIOError as exc:
298                # Ignore some errors, see tickets #51 and #17 at
299                # http://ftputil.sschwarzer.net/trac/ticket/51 and
300                # http://ftputil.sschwarzer.net/trac/ticket/17,
301                # respectively.
302                exc = str(exc)
303                error_code = exc[:3]
304                if exc.splitlines()[0] != "timed out" and \
305                  error_code not in ("150", "426", "450", "451"):
306                    raise
307        finally:
308            # Restore timeout for socket of `_FTPFile`'s `ftplib.FTP`
309            # object in case the connection is reused later.
310            self._session.sock.settimeout(old_timeout)
311            # If something went wrong before, the file is probably
312            # defunct and subsequent calls to `close` won't help
313            # either, so we consider the file closed for practical
314            # purposes.
315            self.closed = True
Note: See TracBrowser for help on using the repository browser.