source: test/test_stat.py @ 1721:3557f65ded13

Last change on this file since 1721:3557f65ded13 was 1721:3557f65ded13, checked in by Stefan Schwarzer <sschwarzer@…>, 8 months ago
Remove `compat.py` This module was for Python 2/3 compatibility, similar to the `six` package. Since the next version of ftputil will only support Python 3, hardcode the types from `compat.py` in the code that used to use the `compat` module. There may still be redundant code that isn't needed when running the tests under Python 3.
File size: 22.7 KB
Line 
1# Copyright (C) 2003-2018, Stefan Schwarzer <sschwarzer@sschwarzer.net>
2# and ftputil contributors (see `doc/contributors.txt`)
3# See the file LICENSE for licensing terms.
4
5import stat
6import time
7
8import pytest
9
10import ftputil
11import ftputil.error
12import ftputil.stat
13from ftputil.stat import MINUTE_PRECISION, DAY_PRECISION, UNKNOWN_PRECISION
14
15from test import test_base
16from test import mock_ftplib
17
18
19def _test_stat(session_factory):
20    host = test_base.ftp_host_factory(session_factory=session_factory)
21    stat = ftputil.stat._Stat(host)
22    # Use Unix format parser explicitly. This doesn't exclude switching
23    # to the MS format parser later if the test allows this switching.
24    stat._parser = ftputil.stat.UnixParser()
25    return stat
26
27
28# Special value to handle special case of datetimes before the epoch.
29EPOCH = time.gmtime(0)[:6]
30
31def stat_tuple_to_seconds(t):
32    """
33    Return a float number representing the local time associated with
34    the six-element tuple `t`.
35    """
36    assert len(t) == 6, \
37             "need a six-element tuple (year, month, day, hour, min, sec)"
38    # Do _not_ apply `time.mktime` to the `EPOCH` value below. On some
39    # platforms (e. g. Windows) this might cause an `OverflowError`.
40    if t == EPOCH:
41        return 0.0
42    else:
43        return time.mktime(t + (0, 0, -1))
44
45
46class TestParsers:
47
48    #
49    # Helper methods
50    #
51    def _test_valid_lines(self, parser_class, lines, expected_stat_results):
52        parser = parser_class()
53        for line, expected_stat_result in zip(lines, expected_stat_results):
54            # Convert to list to compare with the list `expected_stat_results`.
55            parse_result = parser.parse_line(line)
56            stat_result = list(parse_result) + \
57                          [parse_result._st_mtime_precision,
58                           parse_result._st_name,
59                           parse_result._st_target]
60            # Convert time tuple to seconds.
61            expected_stat_result[8] = \
62              stat_tuple_to_seconds(expected_stat_result[8])
63            # Compare lists.
64            assert stat_result == expected_stat_result
65
66    def _test_invalid_lines(self, parser_class, lines):
67        parser = parser_class()
68        for line in lines:
69            with pytest.raises(ftputil.error.ParserError):
70                parser.parse_line(line)
71
72    def _expected_year(self):
73        """
74        Return the expected year for the second line in the
75        listing in `test_valid_unix_lines`.
76        """
77        # If in this year it's after Dec 19, 23:11, use the current
78        # year, else use the previous year. This datetime value
79        # corresponds to the hard-coded value in the string lists
80        # below.
81        now = time.localtime()
82        # We need only month, day, hour and minute.
83        current_time_parts = now[1:5]
84        time_parts_in_listing = (12, 19, 23, 11)
85        if current_time_parts > time_parts_in_listing:
86            return now[0]
87        else:
88            return now[0] - 1
89
90    #
91    # Unix parser
92    #
93    def test_valid_unix_lines(self):
94        lines = [
95          "drwxr-sr-x   2 45854    200           512 May  4  2000 "
96            "chemeng link -> chemeng target",
97          # The year value for this line will change with the actual time.
98          "-rw-r--r--   1 45854    200          4604 Dec 19 23:11 index.html",
99          "drwxr-sr-x   2 45854    200           512 May 29  2000 os2",
100          "----------   2 45854    200           512 May 29  2000 some_file",
101          "lrwxrwxrwx   2 45854    200           512 May 29  2000 osup -> "
102                                                                  "../os2"
103        ]
104        expected_stat_results = [
105          [17901, None, None, 2, "45854", "200", 512, None,
106           (2000, 5, 4, 0, 0, 0), None, DAY_PRECISION,
107           "chemeng link", "chemeng target"],
108          [33188, None, None, 1, "45854", "200", 4604, None,
109           (self._expected_year(), 12, 19, 23, 11, 0), None, MINUTE_PRECISION,
110           "index.html", None],
111          [17901, None, None, 2, "45854", "200", 512, None,
112           (2000, 5, 29, 0, 0, 0), None, DAY_PRECISION,
113           "os2", None],
114          [32768, None, None, 2, "45854", "200", 512, None,
115           (2000, 5, 29, 0, 0, 0), None, DAY_PRECISION,
116           "some_file", None],
117          [41471, None, None, 2, "45854", "200", 512, None,
118           (2000, 5, 29, 0, 0, 0), None, DAY_PRECISION,
119           "osup", "../os2"]
120        ]
121        self._test_valid_lines(ftputil.stat.UnixParser, lines,
122                               expected_stat_results)
123
124    def test_alternative_unix_format(self):
125        # See http://ftputil.sschwarzer.net/trac/ticket/12 for a
126        # description for the need for an alternative format.
127        lines = [
128          "drwxr-sr-x   2   200           512 May  4  2000 "
129            "chemeng link -> chemeng target",
130          # The year value for this line will change with the actual time.
131          "-rw-r--r--   1   200          4604 Dec 19 23:11 index.html",
132          "drwxr-sr-x   2   200           512 May 29  2000 os2",
133          "lrwxrwxrwx   2   200           512 May 29  2000 osup -> ../os2"
134        ]
135        expected_stat_results = [
136          [17901, None, None, 2, None, "200", 512, None,
137           (2000, 5, 4, 0, 0, 0), None, DAY_PRECISION,
138           "chemeng link", "chemeng target"],
139          [33188, None, None, 1, None, "200", 4604, None,
140           (self._expected_year(), 12, 19, 23, 11, 0), None, MINUTE_PRECISION,
141           "index.html", None],
142          [17901, None, None, 2, None, "200", 512, None,
143           (2000, 5, 29, 0, 0, 0), None, DAY_PRECISION,
144           "os2", None],
145          [41471, None, None, 2, None, "200", 512, None,
146           (2000, 5, 29, 0, 0, 0), None, DAY_PRECISION,
147           "osup", "../os2"]
148        ]
149        self._test_valid_lines(ftputil.stat.UnixParser, lines,
150                               expected_stat_results)
151
152    def test_pre_epoch_times_for_unix(self):
153        # See http://ftputil.sschwarzer.net/trac/ticket/83 .
154        # `mirrors.ibiblio.org` returns dates before the "epoch" that
155        # cause an `OverflowError` in `mktime` on some platforms,
156        # e. g. Windows.
157        lines = [
158          "-rw-r--r--   1 45854    200          4604 May  4  1968 index.html",
159          "-rw-r--r--   1 45854    200          4604 Dec 31  1969 index.html",
160          "-rw-r--r--   1 45854    200          4604 May  4  1800 index.html",
161        ]
162        expected_stat_result = \
163          [33188, None, None, 1, "45854", "200", 4604, None,
164           EPOCH, None, UNKNOWN_PRECISION, "index.html", None]
165        # Make shallow copies to avoid converting the time tuple more
166        # than once in _test_valid_lines`.
167        expected_stat_results = [expected_stat_result[:],
168                                 expected_stat_result[:],
169                                 expected_stat_result[:]]
170        self._test_valid_lines(ftputil.stat.UnixParser, lines,
171                               expected_stat_results)
172
173    def test_invalid_unix_lines(self):
174        lines = [
175          # Not intended to be parsed. Should have been filtered out by
176          # `ignores_line`.
177          "total 14",
178          # Invalid month abbreviation
179          "drwxr-sr-x   2 45854    200           512 Max  4  2000 chemeng",
180          # Year value isn't an integer
181          "drwxr-sr-x   2 45854    200           512 May  4  abcd chemeng",
182          # Day value isn't an integer
183          "drwxr-sr-x   2 45854    200           512 May ab  2000 chemeng",
184          # Hour value isn't an integer
185          "-rw-r--r--   1 45854    200          4604 Dec 19 ab:11 index.html",
186          # Minute value isn't an integer
187          "-rw-r--r--   1 45854    200          4604 Dec 19 23:ab index.html",
188          # Day value too large
189          "drwxr-sr-x   2 45854    200           512 May 32  2000 chemeng",
190          # Incomplete mode
191          "drwxr-sr-    2 45854    200           512 May  4  2000 chemeng",
192          # Invalid first letter in mode
193          "xrwxr-sr-x   2 45854    200           512 May  4  2000 chemeng",
194          # Ditto, plus invalid size value
195          "xrwxr-sr-x   2 45854    200           51x May  4  2000 chemeng",
196          # Is this `os1 -> os2` pointing to `os3`, or `os1` pointing
197          # to `os2 -> os3` or the plain name `os1 -> os2 -> os3`? We
198          # don't know, so we consider the line invalid.
199          "drwxr-sr-x   2 45854    200           512 May 29  2000 "
200            "os1 -> os2 -> os3",
201          # Missing name
202          "-rwxr-sr-x   2 45854    200           51x May  4  2000 ",
203        ]
204        self._test_invalid_lines(ftputil.stat.UnixParser, lines)
205
206    #
207    # Microsoft parser
208    #
209    def test_valid_ms_lines_two_digit_year(self):
210        lines = [
211          "07-27-01  11:16AM       <DIR>          Test",
212          "10-23-95  03:25PM       <DIR>          WindowsXP",
213          "07-17-00  02:08PM             12266720 test.exe",
214          "07-17-09  12:08AM             12266720 test.exe",
215          "07-17-09  12:08PM             12266720 test.exe"
216        ]
217        expected_stat_results = [
218          [16640, None, None, None, None, None, None, None,
219           (2001, 7, 27, 11, 16, 0), None, MINUTE_PRECISION,
220           "Test", None],
221          [16640, None, None, None, None, None, None, None,
222           (1995, 10, 23, 15, 25, 0), None, MINUTE_PRECISION,
223           "WindowsXP", None],
224          [33024, None, None, None, None, None, 12266720, None,
225           (2000, 7, 17, 14, 8, 0), None, MINUTE_PRECISION,
226           "test.exe", None],
227          [33024, None, None, None, None, None, 12266720, None,
228           (2009, 7, 17, 0, 8, 0), None, MINUTE_PRECISION,
229           "test.exe", None],
230          [33024, None, None, None, None, None, 12266720, None,
231           (2009, 7, 17, 12, 8, 0), None, MINUTE_PRECISION,
232           "test.exe", None]
233        ]
234        self._test_valid_lines(ftputil.stat.MSParser, lines,
235                               expected_stat_results)
236
237    def test_valid_ms_lines_four_digit_year(self):
238        # See http://ftputil.sschwarzer.net/trac/ticket/67
239        lines = [
240          "10-19-2012  03:13PM       <DIR>          SYNCDEST",
241          "10-19-2012  03:13PM       <DIR>          SYNCSOURCE",
242          "10-19-1968  03:13PM       <DIR>          SYNC"
243        ]
244        expected_stat_results = [
245          [16640, None, None, None, None, None, None, None,
246           (2012, 10, 19, 15, 13, 0), None, MINUTE_PRECISION,
247           "SYNCDEST", None],
248          [16640, None, None, None, None, None, None, None,
249           (2012, 10, 19, 15, 13, 0), None, MINUTE_PRECISION,
250           "SYNCSOURCE", None],
251          [16640, None, None, None, None, None, None, None,
252           EPOCH, None, UNKNOWN_PRECISION,
253           "SYNC", None],
254        ]
255        self._test_valid_lines(ftputil.stat.MSParser, lines,
256                               expected_stat_results)
257
258    def test_invalid_ms_lines(self):
259        lines = [
260          # Neither "<DIR>" nor a size present
261          "07-27-01  11:16AM                      Test",
262          # "AM"/"PM" missing
263          "07-17-00  02:08             12266720 test.exe",
264          # Year not an int
265          "07-17-ab  02:08AM           12266720 test.exe",
266          # Month not an int
267          "ab-17-00  02:08AM           12266720 test.exe",
268          # Day not an int
269          "07-ab-00  02:08AM           12266720 test.exe",
270          # Hour not an int
271          "07-17-00  ab:08AM           12266720 test.exe",
272          # Invalid size value
273          "07-17-00  02:08AM           1226672x test.exe"
274        ]
275        self._test_invalid_lines(ftputil.stat.MSParser, lines)
276
277    #
278    # The following code checks if the decision logic in the Unix
279    # line parser for determining the year works.
280    #
281    def datetime_string(self, time_float):
282        """
283        Return a datetime string generated from the value in
284        `time_float`. The parameter value is a floating point value
285        as returned by `time.time()`. The returned string is built as
286        if it were from a Unix FTP server (format: MMM dd hh:mm")
287        """
288        time_tuple = time.localtime(time_float)
289        return time.strftime("%b %d %H:%M", time_tuple)
290
291    def dir_line(self, time_float):
292        """
293        Return a directory line as from a Unix FTP server. Most of
294        the contents are fixed, but the timestamp is made from
295        `time_float` (seconds since the epoch, as from `time.time()`).
296        """
297        line_template = \
298          "-rw-r--r--   1   45854   200   4604   {0}   index.html"
299        return line_template.format(self.datetime_string(time_float))
300
301    def assert_equal_times(self, time1, time2):
302        """
303        Check if both times (seconds since the epoch) are equal. For
304        the purpose of this test, two times are "equal" if they
305        differ no more than one minute from each other.
306        """
307        abs_difference = abs(time1 - time2)
308        assert abs_difference <= 60.0, \
309                 "Difference is %s seconds" % abs_difference
310
311    def _test_time_shift(self, supposed_time_shift, deviation=0.0):
312        """
313        Check if the stat parser considers the time shift value
314        correctly. `deviation` is the difference between the actual
315        time shift and the supposed time shift, which is rounded
316        to full hours.
317        """
318        host = test_base.ftp_host_factory()
319        # Explicitly use Unix format parser here.
320        host._stat._parser = ftputil.stat.UnixParser()
321        host.set_time_shift(supposed_time_shift)
322        server_time = time.time() + supposed_time_shift + deviation
323        stat_result = host._stat._parser.parse_line(self.dir_line(server_time),
324                                                    host.time_shift())
325        self.assert_equal_times(stat_result.st_mtime, server_time)
326
327    def test_time_shifts(self):
328        """Test correct year depending on time shift value."""
329        # 1. test: Client and server share the same local time
330        self._test_time_shift(0.0)
331        # 2. test: Server is three hours ahead of client
332        self._test_time_shift(3 * 60 * 60)
333        # 3. test: Client is three hours ahead of server
334        self._test_time_shift(- 3 * 60 * 60)
335        # 4. test: Server is supposed to be three hours ahead, but
336        #    is ahead three hours and one minute
337        self._test_time_shift(3 * 60 * 60, 60)
338        # 5. test: Server is supposed to be three hours ahead, but
339        #    is ahead three hours minus one minute
340        self._test_time_shift(3 * 60 * 60, -60)
341        # 6. test: Client is supposed to be three hours ahead, but
342        #    is ahead three hours and one minute
343        self._test_time_shift(-3 * 60 * 60, -60)
344        # 7. test: Client is supposed to be three hours ahead, but
345        #    is ahead three hours minus one minute
346        self._test_time_shift(-3 * 60 * 60, 60)
347
348
349class TestLstatAndStat:
350    """
351    Test `FTPHost.lstat` and `FTPHost.stat` (test currently only
352    implemented for Unix server format).
353    """
354
355    def setup_method(self, method):
356        # Most tests in this class need the mock session class with
357        # Unix format, so make this the default. Tests which need
358        # the MS format can overwrite `self.stat` later.
359        self.stat = \
360          _test_stat(session_factory=mock_ftplib.MockUnixFormatSession)
361
362    def test_repr(self):
363        """Test if the `repr` result looks like a named tuple."""
364        stat_result = self.stat._lstat("/home/sschwarzer/chemeng")
365        # TODO: Make the value for `st_mtime` robust against DST "time
366        # zone" changes.
367        expected_result = (
368          "StatResult(st_mode=17901, st_ino=None, st_dev=None, "
369          "st_nlink=2, st_uid='45854', st_gid='200', st_size=512, "
370          "st_atime=None, st_mtime=957391200.0, st_ctime=None)")
371        assert repr(stat_result) == expected_result
372
373    def test_failing_lstat(self):
374        """Test whether `lstat` fails for a nonexistent path."""
375        with pytest.raises(ftputil.error.PermanentError):
376            self.stat._lstat("/home/sschw/notthere")
377        with pytest.raises(ftputil.error.PermanentError):
378            self.stat._lstat("/home/sschwarzer/notthere")
379
380    def test_lstat_for_root(self):
381        """
382        Test `lstat` for `/` .
383
384        Note: `(l)stat` works by going one directory up and parsing
385        the output of an FTP `LIST` command. Unfortunately, it's not
386        possible to do this for the root directory `/`.
387        """
388        with pytest.raises(ftputil.error.RootDirError) as exc_info:
389            self.stat._lstat("/")
390        # `RootDirError` is "outside" the `FTPOSError` hierarchy.
391        assert not isinstance(exc_info.value, ftputil.error.FTPOSError)
392        del exc_info
393
394    def test_lstat_one_unix_file(self):
395        """Test `lstat` for a file described in Unix-style format."""
396        stat_result = self.stat._lstat("/home/sschwarzer/index.html")
397        # Second form is needed for Python 3
398        assert oct(stat_result.st_mode) in ("0100644", "0o100644")
399        assert stat_result.st_size == 4604
400        assert stat_result._st_mtime_precision == 60
401
402    def test_lstat_one_ms_file(self):
403        """Test `lstat` for a file described in DOS-style format."""
404        self.stat = _test_stat(session_factory=mock_ftplib.MockMSFormatSession)
405        stat_result = self.stat._lstat("/home/msformat/abcd.exe")
406        assert stat_result._st_mtime_precision == 60
407
408    def test_lstat_one_unix_dir(self):
409        """Test `lstat` for a directory described in Unix-style format."""
410        stat_result = self.stat._lstat("/home/sschwarzer/scios2")
411        # Second form is needed for Python 3
412        assert oct(stat_result.st_mode) in ("042755", "0o42755")
413        assert stat_result.st_ino is None
414        assert stat_result.st_dev is None
415        assert stat_result.st_nlink == 6
416        assert stat_result.st_uid == "45854"
417        assert stat_result.st_gid == "200"
418        assert stat_result.st_size == 512
419        assert stat_result.st_atime is None
420        assert (stat_result.st_mtime ==
421                stat_tuple_to_seconds((1999, 9, 20, 0, 0, 0)))
422        assert stat_result.st_ctime is None
423        assert stat_result._st_mtime_precision == 24*60*60
424        assert stat_result == (17901, None, None, 6, "45854", "200", 512, None,
425                               stat_tuple_to_seconds((1999, 9, 20, 0, 0, 0)),
426                               None)
427
428    def test_lstat_one_ms_dir(self):
429        """Test `lstat` for a directory described in DOS-style format."""
430        self.stat = _test_stat(session_factory=mock_ftplib.MockMSFormatSession)
431        stat_result = self.stat._lstat("/home/msformat/WindowsXP")
432        assert stat_result._st_mtime_precision == 60
433
434    def test_lstat_via_stat_module(self):
435        """Test `lstat` indirectly via `stat` module."""
436        stat_result = self.stat._lstat("/home/sschwarzer/")
437        assert stat.S_ISDIR(stat_result.st_mode)
438
439    def test_stat_following_link(self):
440        """Test `stat` when invoked on a link."""
441        # Simple link
442        stat_result = self.stat._stat("/home/link")
443        assert stat_result.st_size == 4604
444        # Link pointing to a link
445        stat_result = self.stat._stat("/home/python/link_link")
446        assert stat_result.st_size == 4604
447        stat_result = self.stat._stat("../python/link_link")
448        assert stat_result.st_size == 4604
449        # Recursive link structures
450        with pytest.raises(ftputil.error.PermanentError):
451            self.stat._stat("../python/bad_link")
452        with pytest.raises(ftputil.error.PermanentError):
453            self.stat._stat("/home/bad_link")
454
455    #
456    # Test automatic switching of Unix/MS parsers
457    #
458    def test_parser_switching_with_permanent_error(self):
459        """Test non-switching of parser format with `PermanentError`."""
460        self.stat = _test_stat(session_factory=mock_ftplib.MockMSFormatSession)
461        assert self.stat._allow_parser_switching is True
462        # With these directory contents, we get a `ParserError` for
463        # the Unix parser first, so `_allow_parser_switching` can be
464        # switched off no matter whether we got a `PermanentError`
465        # afterward or not.
466        with pytest.raises(ftputil.error.PermanentError):
467            self.stat._lstat("/home/msformat/nonexistent")
468        assert self.stat._allow_parser_switching is False
469
470    def test_parser_switching_default_to_unix(self):
471        """Test non-switching of parser format; stay with Unix."""
472        assert self.stat._allow_parser_switching is True
473        assert isinstance(self.stat._parser, ftputil.stat.UnixParser)
474        stat_result = self.stat._lstat("/home/sschwarzer/index.html")
475        # The Unix parser worked, so keep it.
476        assert isinstance(self.stat._parser, ftputil.stat.UnixParser)
477        assert self.stat._allow_parser_switching is False
478
479    def test_parser_switching_to_ms(self):
480        """Test switching of parser from Unix to MS format."""
481        self.stat = _test_stat(session_factory=mock_ftplib.MockMSFormatSession)
482        assert self.stat._allow_parser_switching is True
483        assert isinstance(self.stat._parser, ftputil.stat.UnixParser)
484        # Parsing the directory `/home/msformat` with the Unix parser
485        # fails, so switch to the MS parser.
486        stat_result = self.stat._lstat("/home/msformat/abcd.exe")
487        assert isinstance(self.stat._parser, ftputil.stat.MSParser)
488        assert self.stat._allow_parser_switching is False
489        assert stat_result._st_name == "abcd.exe"
490        assert stat_result.st_size == 12266720
491
492    def test_parser_switching_regarding_empty_dir(self):
493        """Test switching of parser if a directory is empty."""
494        self.stat = _test_stat(session_factory=mock_ftplib.MockMSFormatSession)
495        assert self.stat._allow_parser_switching is True
496        # When the directory we're looking into doesn't give us any
497        # lines we can't decide whether the first parser worked,
498        # because it wasn't applied. So keep the parser for now.
499        result = self.stat._listdir("/home/msformat/XPLaunch/empty")
500        assert result == []
501        assert self.stat._allow_parser_switching is True
502        assert isinstance(self.stat._parser, ftputil.stat.UnixParser)
503
504
505class TestListdir:
506    """Test `FTPHost.listdir`."""
507
508    def setup_method(self, method):
509        self.stat = \
510          _test_stat(session_factory=mock_ftplib.MockUnixFormatSession)
511
512    def test_failing_listdir(self):
513        """Test failing `FTPHost.listdir`."""
514        with pytest.raises(ftputil.error.PermanentError):
515            self.stat._listdir("notthere")
516
517    def test_succeeding_listdir(self):
518        """Test succeeding `FTPHost.listdir`."""
519        # Do we have all expected "files"?
520        assert len(self.stat._listdir(".")) == 9
521        # Have they the expected names?
522        expected = ("chemeng download image index.html os2 "
523                    "osup publications python scios2").split()
524        remote_file_list = self.stat._listdir(".")
525        for file in expected:
526            assert file in remote_file_list
Note: See TracBrowser for help on using the repository browser.