source: test/test_stat.py @ 1718:8bed138bc404

Last change on this file since 1718:8bed138bc404 was 1718:8bed138bc404, checked in by Stefan Schwarzer <sschwarzer@…>, 4 months ago
Don't inherit from `object` This is no longer needed because we're targeting Python 3 only now.
File size: 23.1 KB
Line 
1# Copyright (C) 2003-2018, Stefan Schwarzer <sschwarzer@sschwarzer.net>
2# and ftputil contributors (see `doc/contributors.txt`)
3# See the file LICENSE for licensing terms.
4
5import stat
6import time
7
8import pytest
9
10import ftputil
11import ftputil.compat
12import ftputil.error
13import ftputil.stat
14from ftputil.stat import MINUTE_PRECISION, DAY_PRECISION, UNKNOWN_PRECISION
15
16from test import test_base
17from test import mock_ftplib
18
19
20def _test_stat(session_factory):
21    host = test_base.ftp_host_factory(session_factory=session_factory)
22    stat = ftputil.stat._Stat(host)
23    # Use Unix format parser explicitly. This doesn't exclude switching
24    # to the MS format parser later if the test allows this switching.
25    stat._parser = ftputil.stat.UnixParser()
26    return stat
27
28
29# Special value to handle special case of datetimes before the epoch.
30EPOCH = time.gmtime(0)[:6]
31
32def stat_tuple_to_seconds(t):
33    """
34    Return a float number representing the local time associated with
35    the six-element tuple `t`.
36    """
37    assert len(t) == 6, \
38             "need a six-element tuple (year, month, day, hour, min, sec)"
39    # Do _not_ apply `time.mktime` to the `EPOCH` value below. On some
40    # platforms (e. g. Windows) this might cause an `OverflowError`.
41    if t == EPOCH:
42        return 0.0
43    else:
44        return time.mktime(t + (0, 0, -1))
45
46
47class TestParsers:
48
49    #
50    # Helper methods
51    #
52    def _test_valid_lines(self, parser_class, lines, expected_stat_results):
53        parser = parser_class()
54        for line, expected_stat_result in zip(lines, expected_stat_results):
55            # Convert to list to compare with the list `expected_stat_results`.
56            parse_result = parser.parse_line(line)
57            stat_result = list(parse_result) + \
58                          [parse_result._st_mtime_precision,
59                           parse_result._st_name,
60                           parse_result._st_target]
61            # Convert time tuple to seconds.
62            expected_stat_result[8] = \
63              stat_tuple_to_seconds(expected_stat_result[8])
64            # Compare lists.
65            assert stat_result == expected_stat_result
66
67    def _test_invalid_lines(self, parser_class, lines):
68        parser = parser_class()
69        for line in lines:
70            with pytest.raises(ftputil.error.ParserError):
71                parser.parse_line(line)
72
73    def _expected_year(self):
74        """
75        Return the expected year for the second line in the
76        listing in `test_valid_unix_lines`.
77        """
78        # If in this year it's after Dec 19, 23:11, use the current
79        # year, else use the previous year. This datetime value
80        # corresponds to the hard-coded value in the string lists
81        # below.
82        now = time.localtime()
83        # We need only month, day, hour and minute.
84        current_time_parts = now[1:5]
85        time_parts_in_listing = (12, 19, 23, 11)
86        if current_time_parts > time_parts_in_listing:
87            return now[0]
88        else:
89            return now[0] - 1
90
91    #
92    # Unix parser
93    #
94    def test_valid_unix_lines(self):
95        lines = [
96          "drwxr-sr-x   2 45854    200           512 May  4  2000 "
97            "chemeng link -> chemeng target",
98          # The year value for this line will change with the actual time.
99          "-rw-r--r--   1 45854    200          4604 Dec 19 23:11 index.html",
100          "drwxr-sr-x   2 45854    200           512 May 29  2000 os2",
101          "----------   2 45854    200           512 May 29  2000 some_file",
102          "lrwxrwxrwx   2 45854    200           512 May 29  2000 osup -> "
103                                                                  "../os2"
104        ]
105        expected_stat_results = [
106          [17901, None, None, 2, "45854", "200", 512, None,
107           (2000, 5, 4, 0, 0, 0), None, DAY_PRECISION,
108           "chemeng link", "chemeng target"],
109          [33188, None, None, 1, "45854", "200", 4604, None,
110           (self._expected_year(), 12, 19, 23, 11, 0), None, MINUTE_PRECISION,
111           "index.html", None],
112          [17901, None, None, 2, "45854", "200", 512, None,
113           (2000, 5, 29, 0, 0, 0), None, DAY_PRECISION,
114           "os2", None],
115          [32768, None, None, 2, "45854", "200", 512, None,
116           (2000, 5, 29, 0, 0, 0), None, DAY_PRECISION,
117           "some_file", None],
118          [41471, None, None, 2, "45854", "200", 512, None,
119           (2000, 5, 29, 0, 0, 0), None, DAY_PRECISION,
120           "osup", "../os2"]
121        ]
122        self._test_valid_lines(ftputil.stat.UnixParser, lines,
123                               expected_stat_results)
124
125    def test_alternative_unix_format(self):
126        # See http://ftputil.sschwarzer.net/trac/ticket/12 for a
127        # description for the need for an alternative format.
128        lines = [
129          "drwxr-sr-x   2   200           512 May  4  2000 "
130            "chemeng link -> chemeng target",
131          # The year value for this line will change with the actual time.
132          "-rw-r--r--   1   200          4604 Dec 19 23:11 index.html",
133          "drwxr-sr-x   2   200           512 May 29  2000 os2",
134          "lrwxrwxrwx   2   200           512 May 29  2000 osup -> ../os2"
135        ]
136        expected_stat_results = [
137          [17901, None, None, 2, None, "200", 512, None,
138           (2000, 5, 4, 0, 0, 0), None, DAY_PRECISION,
139           "chemeng link", "chemeng target"],
140          [33188, None, None, 1, None, "200", 4604, None,
141           (self._expected_year(), 12, 19, 23, 11, 0), None, MINUTE_PRECISION,
142           "index.html", None],
143          [17901, None, None, 2, None, "200", 512, None,
144           (2000, 5, 29, 0, 0, 0), None, DAY_PRECISION,
145           "os2", None],
146          [41471, None, None, 2, None, "200", 512, None,
147           (2000, 5, 29, 0, 0, 0), None, DAY_PRECISION,
148           "osup", "../os2"]
149        ]
150        self._test_valid_lines(ftputil.stat.UnixParser, lines,
151                               expected_stat_results)
152
153    def test_pre_epoch_times_for_unix(self):
154        # See http://ftputil.sschwarzer.net/trac/ticket/83 .
155        # `mirrors.ibiblio.org` returns dates before the "epoch" that
156        # cause an `OverflowError` in `mktime` on some platforms,
157        # e. g. Windows.
158        lines = [
159          "-rw-r--r--   1 45854    200          4604 May  4  1968 index.html",
160          "-rw-r--r--   1 45854    200          4604 Dec 31  1969 index.html",
161          "-rw-r--r--   1 45854    200          4604 May  4  1800 index.html",
162        ]
163        expected_stat_result = \
164          [33188, None, None, 1, "45854", "200", 4604, None,
165           EPOCH, None, UNKNOWN_PRECISION, "index.html", None]
166        # Make shallow copies to avoid converting the time tuple more
167        # than once in _test_valid_lines`.
168        expected_stat_results = [expected_stat_result[:],
169                                 expected_stat_result[:],
170                                 expected_stat_result[:]]
171        self._test_valid_lines(ftputil.stat.UnixParser, lines,
172                               expected_stat_results)
173
174    def test_invalid_unix_lines(self):
175        lines = [
176          # Not intended to be parsed. Should have been filtered out by
177          # `ignores_line`.
178          "total 14",
179          # Invalid month abbreviation
180          "drwxr-sr-x   2 45854    200           512 Max  4  2000 chemeng",
181          # Year value isn't an integer
182          "drwxr-sr-x   2 45854    200           512 May  4  abcd chemeng",
183          # Day value isn't an integer
184          "drwxr-sr-x   2 45854    200           512 May ab  2000 chemeng",
185          # Hour value isn't an integer
186          "-rw-r--r--   1 45854    200          4604 Dec 19 ab:11 index.html",
187          # Minute value isn't an integer
188          "-rw-r--r--   1 45854    200          4604 Dec 19 23:ab index.html",
189          # Day value too large
190          "drwxr-sr-x   2 45854    200           512 May 32  2000 chemeng",
191          # Incomplete mode
192          "drwxr-sr-    2 45854    200           512 May  4  2000 chemeng",
193          # Invalid first letter in mode
194          "xrwxr-sr-x   2 45854    200           512 May  4  2000 chemeng",
195          # Ditto, plus invalid size value
196          "xrwxr-sr-x   2 45854    200           51x May  4  2000 chemeng",
197          # Is this `os1 -> os2` pointing to `os3`, or `os1` pointing
198          # to `os2 -> os3` or the plain name `os1 -> os2 -> os3`? We
199          # don't know, so we consider the line invalid.
200          "drwxr-sr-x   2 45854    200           512 May 29  2000 "
201            "os1 -> os2 -> os3",
202          # Missing name
203          "-rwxr-sr-x   2 45854    200           51x May  4  2000 ",
204        ]
205        self._test_invalid_lines(ftputil.stat.UnixParser, lines)
206
207    #
208    # Microsoft parser
209    #
210    def test_valid_ms_lines_two_digit_year(self):
211        lines = [
212          "07-27-01  11:16AM       <DIR>          Test",
213          "10-23-95  03:25PM       <DIR>          WindowsXP",
214          "07-17-00  02:08PM             12266720 test.exe",
215          "07-17-09  12:08AM             12266720 test.exe",
216          "07-17-09  12:08PM             12266720 test.exe"
217        ]
218        expected_stat_results = [
219          [16640, None, None, None, None, None, None, None,
220           (2001, 7, 27, 11, 16, 0), None, MINUTE_PRECISION,
221           "Test", None],
222          [16640, None, None, None, None, None, None, None,
223           (1995, 10, 23, 15, 25, 0), None, MINUTE_PRECISION,
224           "WindowsXP", None],
225          [33024, None, None, None, None, None, 12266720, None,
226           (2000, 7, 17, 14, 8, 0), None, MINUTE_PRECISION,
227           "test.exe", None],
228          [33024, None, None, None, None, None, 12266720, None,
229           (2009, 7, 17, 0, 8, 0), None, MINUTE_PRECISION,
230           "test.exe", None],
231          [33024, None, None, None, None, None, 12266720, None,
232           (2009, 7, 17, 12, 8, 0), None, MINUTE_PRECISION,
233           "test.exe", None]
234        ]
235        self._test_valid_lines(ftputil.stat.MSParser, lines,
236                               expected_stat_results)
237
238    def test_valid_ms_lines_four_digit_year(self):
239        # See http://ftputil.sschwarzer.net/trac/ticket/67
240        lines = [
241          "10-19-2012  03:13PM       <DIR>          SYNCDEST",
242          "10-19-2012  03:13PM       <DIR>          SYNCSOURCE",
243          "10-19-1968  03:13PM       <DIR>          SYNC"
244        ]
245        expected_stat_results = [
246          [16640, None, None, None, None, None, None, None,
247           (2012, 10, 19, 15, 13, 0), None, MINUTE_PRECISION,
248           "SYNCDEST", None],
249          [16640, None, None, None, None, None, None, None,
250           (2012, 10, 19, 15, 13, 0), None, MINUTE_PRECISION,
251           "SYNCSOURCE", None],
252          [16640, None, None, None, None, None, None, None,
253           EPOCH, None, UNKNOWN_PRECISION,
254           "SYNC", None],
255        ]
256        self._test_valid_lines(ftputil.stat.MSParser, lines,
257                               expected_stat_results)
258
259    def test_invalid_ms_lines(self):
260        lines = [
261          # Neither "<DIR>" nor a size present
262          "07-27-01  11:16AM                      Test",
263          # "AM"/"PM" missing
264          "07-17-00  02:08             12266720 test.exe",
265          # Year not an int
266          "07-17-ab  02:08AM           12266720 test.exe",
267          # Month not an int
268          "ab-17-00  02:08AM           12266720 test.exe",
269          # Day not an int
270          "07-ab-00  02:08AM           12266720 test.exe",
271          # Hour not an int
272          "07-17-00  ab:08AM           12266720 test.exe",
273          # Invalid size value
274          "07-17-00  02:08AM           1226672x test.exe"
275        ]
276        self._test_invalid_lines(ftputil.stat.MSParser, lines)
277
278    #
279    # The following code checks if the decision logic in the Unix
280    # line parser for determining the year works.
281    #
282    def datetime_string(self, time_float):
283        """
284        Return a datetime string generated from the value in
285        `time_float`. The parameter value is a floating point value
286        as returned by `time.time()`. The returned string is built as
287        if it were from a Unix FTP server (format: MMM dd hh:mm")
288        """
289        time_tuple = time.localtime(time_float)
290        return time.strftime("%b %d %H:%M", time_tuple)
291
292    def dir_line(self, time_float):
293        """
294        Return a directory line as from a Unix FTP server. Most of
295        the contents are fixed, but the timestamp is made from
296        `time_float` (seconds since the epoch, as from `time.time()`).
297        """
298        line_template = \
299          "-rw-r--r--   1   45854   200   4604   {0}   index.html"
300        return line_template.format(self.datetime_string(time_float))
301
302    def assert_equal_times(self, time1, time2):
303        """
304        Check if both times (seconds since the epoch) are equal. For
305        the purpose of this test, two times are "equal" if they
306        differ no more than one minute from each other.
307        """
308        abs_difference = abs(time1 - time2)
309        assert abs_difference <= 60.0, \
310                 "Difference is %s seconds" % abs_difference
311
312    def _test_time_shift(self, supposed_time_shift, deviation=0.0):
313        """
314        Check if the stat parser considers the time shift value
315        correctly. `deviation` is the difference between the actual
316        time shift and the supposed time shift, which is rounded
317        to full hours.
318        """
319        host = test_base.ftp_host_factory()
320        # Explicitly use Unix format parser here.
321        host._stat._parser = ftputil.stat.UnixParser()
322        host.set_time_shift(supposed_time_shift)
323        server_time = time.time() + supposed_time_shift + deviation
324        stat_result = host._stat._parser.parse_line(self.dir_line(server_time),
325                                                    host.time_shift())
326        self.assert_equal_times(stat_result.st_mtime, server_time)
327
328    def test_time_shifts(self):
329        """Test correct year depending on time shift value."""
330        # 1. test: Client and server share the same local time
331        self._test_time_shift(0.0)
332        # 2. test: Server is three hours ahead of client
333        self._test_time_shift(3 * 60 * 60)
334        # 3. test: Client is three hours ahead of server
335        self._test_time_shift(- 3 * 60 * 60)
336        # 4. test: Server is supposed to be three hours ahead, but
337        #    is ahead three hours and one minute
338        self._test_time_shift(3 * 60 * 60, 60)
339        # 5. test: Server is supposed to be three hours ahead, but
340        #    is ahead three hours minus one minute
341        self._test_time_shift(3 * 60 * 60, -60)
342        # 6. test: Client is supposed to be three hours ahead, but
343        #    is ahead three hours and one minute
344        self._test_time_shift(-3 * 60 * 60, -60)
345        # 7. test: Client is supposed to be three hours ahead, but
346        #    is ahead three hours minus one minute
347        self._test_time_shift(-3 * 60 * 60, 60)
348
349
350class TestLstatAndStat:
351    """
352    Test `FTPHost.lstat` and `FTPHost.stat` (test currently only
353    implemented for Unix server format).
354    """
355
356    def setup_method(self, method):
357        # Most tests in this class need the mock session class with
358        # Unix format, so make this the default. Tests which need
359        # the MS format can overwrite `self.stat` later.
360        self.stat = \
361          _test_stat(session_factory=mock_ftplib.MockUnixFormatSession)
362
363    def test_repr(self):
364        """Test if the `repr` result looks like a named tuple."""
365        stat_result = self.stat._lstat("/home/sschwarzer/chemeng")
366        # Only under Python 2, unicode strings have the `u` prefix.
367        # TODO: Make the value for `st_mtime` robust against DST "time
368        # zone" changes.
369        if ftputil.compat.python_version == 2:
370            expected_result = (
371              b"StatResult(st_mode=17901, st_ino=None, st_dev=None, "
372              b"st_nlink=2, st_uid=u'45854', st_gid=u'200', st_size=512, "
373              b"st_atime=None, st_mtime=957391200.0, st_ctime=None)")
374        else:
375            expected_result = (
376              "StatResult(st_mode=17901, st_ino=None, st_dev=None, "
377              "st_nlink=2, st_uid='45854', st_gid='200', st_size=512, "
378              "st_atime=None, st_mtime=957391200.0, st_ctime=None)")
379        assert repr(stat_result) == expected_result
380
381    def test_failing_lstat(self):
382        """Test whether `lstat` fails for a nonexistent path."""
383        with pytest.raises(ftputil.error.PermanentError):
384            self.stat._lstat("/home/sschw/notthere")
385        with pytest.raises(ftputil.error.PermanentError):
386            self.stat._lstat("/home/sschwarzer/notthere")
387
388    def test_lstat_for_root(self):
389        """
390        Test `lstat` for `/` .
391
392        Note: `(l)stat` works by going one directory up and parsing
393        the output of an FTP `LIST` command. Unfortunately, it's not
394        possible to do this for the root directory `/`.
395        """
396        with pytest.raises(ftputil.error.RootDirError) as exc_info:
397            self.stat._lstat("/")
398        # `RootDirError` is "outside" the `FTPOSError` hierarchy.
399        assert not isinstance(exc_info.value, ftputil.error.FTPOSError)
400        del exc_info
401
402    def test_lstat_one_unix_file(self):
403        """Test `lstat` for a file described in Unix-style format."""
404        stat_result = self.stat._lstat("/home/sschwarzer/index.html")
405        # Second form is needed for Python 3
406        assert oct(stat_result.st_mode) in ("0100644", "0o100644")
407        assert stat_result.st_size == 4604
408        assert stat_result._st_mtime_precision == 60
409
410    def test_lstat_one_ms_file(self):
411        """Test `lstat` for a file described in DOS-style format."""
412        self.stat = _test_stat(session_factory=mock_ftplib.MockMSFormatSession)
413        stat_result = self.stat._lstat("/home/msformat/abcd.exe")
414        assert stat_result._st_mtime_precision == 60
415
416    def test_lstat_one_unix_dir(self):
417        """Test `lstat` for a directory described in Unix-style format."""
418        stat_result = self.stat._lstat("/home/sschwarzer/scios2")
419        # Second form is needed for Python 3
420        assert oct(stat_result.st_mode) in ("042755", "0o42755")
421        assert stat_result.st_ino is None
422        assert stat_result.st_dev is None
423        assert stat_result.st_nlink == 6
424        assert stat_result.st_uid == "45854"
425        assert stat_result.st_gid == "200"
426        assert stat_result.st_size == 512
427        assert stat_result.st_atime is None
428        assert (stat_result.st_mtime ==
429                stat_tuple_to_seconds((1999, 9, 20, 0, 0, 0)))
430        assert stat_result.st_ctime is None
431        assert stat_result._st_mtime_precision == 24*60*60
432        assert stat_result == (17901, None, None, 6, "45854", "200", 512, None,
433                               stat_tuple_to_seconds((1999, 9, 20, 0, 0, 0)),
434                               None)
435
436    def test_lstat_one_ms_dir(self):
437        """Test `lstat` for a directory described in DOS-style format."""
438        self.stat = _test_stat(session_factory=mock_ftplib.MockMSFormatSession)
439        stat_result = self.stat._lstat("/home/msformat/WindowsXP")
440        assert stat_result._st_mtime_precision == 60
441
442    def test_lstat_via_stat_module(self):
443        """Test `lstat` indirectly via `stat` module."""
444        stat_result = self.stat._lstat("/home/sschwarzer/")
445        assert stat.S_ISDIR(stat_result.st_mode)
446
447    def test_stat_following_link(self):
448        """Test `stat` when invoked on a link."""
449        # Simple link
450        stat_result = self.stat._stat("/home/link")
451        assert stat_result.st_size == 4604
452        # Link pointing to a link
453        stat_result = self.stat._stat("/home/python/link_link")
454        assert stat_result.st_size == 4604
455        stat_result = self.stat._stat("../python/link_link")
456        assert stat_result.st_size == 4604
457        # Recursive link structures
458        with pytest.raises(ftputil.error.PermanentError):
459            self.stat._stat("../python/bad_link")
460        with pytest.raises(ftputil.error.PermanentError):
461            self.stat._stat("/home/bad_link")
462
463    #
464    # Test automatic switching of Unix/MS parsers
465    #
466    def test_parser_switching_with_permanent_error(self):
467        """Test non-switching of parser format with `PermanentError`."""
468        self.stat = _test_stat(session_factory=mock_ftplib.MockMSFormatSession)
469        assert self.stat._allow_parser_switching is True
470        # With these directory contents, we get a `ParserError` for
471        # the Unix parser first, so `_allow_parser_switching` can be
472        # switched off no matter whether we got a `PermanentError`
473        # afterward or not.
474        with pytest.raises(ftputil.error.PermanentError):
475            self.stat._lstat("/home/msformat/nonexistent")
476        assert self.stat._allow_parser_switching is False
477
478    def test_parser_switching_default_to_unix(self):
479        """Test non-switching of parser format; stay with Unix."""
480        assert self.stat._allow_parser_switching is True
481        assert isinstance(self.stat._parser, ftputil.stat.UnixParser)
482        stat_result = self.stat._lstat("/home/sschwarzer/index.html")
483        # The Unix parser worked, so keep it.
484        assert isinstance(self.stat._parser, ftputil.stat.UnixParser)
485        assert self.stat._allow_parser_switching is False
486
487    def test_parser_switching_to_ms(self):
488        """Test switching of parser from Unix to MS format."""
489        self.stat = _test_stat(session_factory=mock_ftplib.MockMSFormatSession)
490        assert self.stat._allow_parser_switching is True
491        assert isinstance(self.stat._parser, ftputil.stat.UnixParser)
492        # Parsing the directory `/home/msformat` with the Unix parser
493        # fails, so switch to the MS parser.
494        stat_result = self.stat._lstat("/home/msformat/abcd.exe")
495        assert isinstance(self.stat._parser, ftputil.stat.MSParser)
496        assert self.stat._allow_parser_switching is False
497        assert stat_result._st_name == "abcd.exe"
498        assert stat_result.st_size == 12266720
499
500    def test_parser_switching_regarding_empty_dir(self):
501        """Test switching of parser if a directory is empty."""
502        self.stat = _test_stat(session_factory=mock_ftplib.MockMSFormatSession)
503        assert self.stat._allow_parser_switching is True
504        # When the directory we're looking into doesn't give us any
505        # lines we can't decide whether the first parser worked,
506        # because it wasn't applied. So keep the parser for now.
507        result = self.stat._listdir("/home/msformat/XPLaunch/empty")
508        assert result == []
509        assert self.stat._allow_parser_switching is True
510        assert isinstance(self.stat._parser, ftputil.stat.UnixParser)
511
512
513class TestListdir:
514    """Test `FTPHost.listdir`."""
515
516    def setup_method(self, method):
517        self.stat = \
518          _test_stat(session_factory=mock_ftplib.MockUnixFormatSession)
519
520    def test_failing_listdir(self):
521        """Test failing `FTPHost.listdir`."""
522        with pytest.raises(ftputil.error.PermanentError):
523            self.stat._listdir("notthere")
524
525    def test_succeeding_listdir(self):
526        """Test succeeding `FTPHost.listdir`."""
527        # Do we have all expected "files"?
528        assert len(self.stat._listdir(".")) == 9
529        # Have they the expected names?
530        expected = ("chemeng download image index.html os2 "
531                    "osup publications python scios2").split()
532        remote_file_list = self.stat._listdir(".")
533        for file in expected:
534            assert file in remote_file_list
Note: See TracBrowser for help on using the repository browser.