source: test/test_stat.py @ 1834:1fd91d46a098

Last change on this file since 1834:1fd91d46a098 was 1834:1fd91d46a098, checked in by Stefan Schwarzer <sschwarzer@…>, 4 months ago
Use `ScriptedSession` in `TestParsers`
File size: 23.0 KB
Line 
1# Copyright (C) 2003-2019, Stefan Schwarzer <sschwarzer@sschwarzer.net>
2# and ftputil contributors (see `doc/contributors.txt`)
3# See the file LICENSE for licensing terms.
4
5import stat
6import time
7
8import pytest
9
10import ftputil
11import ftputil.error
12import ftputil.stat
13from ftputil.stat import MINUTE_PRECISION, DAY_PRECISION, UNKNOWN_PRECISION
14
15from test import test_base
16from test import mock_ftplib
17from test import scripted_session
18
19
20Call = scripted_session.Call
21
22
23def _test_stat(session_factory):
24    host = test_base.ftp_host_factory(session_factory=session_factory)
25    stat = ftputil.stat._Stat(host)
26    # Use Unix format parser explicitly. This doesn't exclude switching
27    # to the MS format parser later if the test allows this switching.
28    stat._parser = ftputil.stat.UnixParser()
29    return stat
30
31
32# Special value to handle special case of datetimes before the epoch.
33EPOCH = time.gmtime(0)[:6]
34
35def stat_tuple_to_seconds(t):
36    """
37    Return a float number representing the local time associated with
38    the six-element tuple `t`.
39    """
40    assert len(t) == 6, \
41             "need a six-element tuple (year, month, day, hour, min, sec)"
42    # Do _not_ apply `time.mktime` to the `EPOCH` value below. On some
43    # platforms (e. g. Windows) this might cause an `OverflowError`.
44    if t == EPOCH:
45        return 0.0
46    else:
47        return time.mktime(t + (0, 0, -1))
48
49
50class TestParsers:
51
52    #
53    # Helper methods
54    #
55    def _test_valid_lines(self, parser_class, lines, expected_stat_results):
56        parser = parser_class()
57        for line, expected_stat_result in zip(lines, expected_stat_results):
58            # Convert to list to compare with the list `expected_stat_results`.
59            parse_result = parser.parse_line(line)
60            stat_result = list(parse_result) + \
61                          [parse_result._st_mtime_precision,
62                           parse_result._st_name,
63                           parse_result._st_target]
64            # Convert time tuple to seconds.
65            expected_stat_result[8] = \
66              stat_tuple_to_seconds(expected_stat_result[8])
67            # Compare lists.
68            assert stat_result == expected_stat_result
69
70    def _test_invalid_lines(self, parser_class, lines):
71        parser = parser_class()
72        for line in lines:
73            with pytest.raises(ftputil.error.ParserError):
74                parser.parse_line(line)
75
76    def _expected_year(self):
77        """
78        Return the expected year for the second line in the
79        listing in `test_valid_unix_lines`.
80        """
81        # If in this year it's after Dec 19, 23:11, use the current
82        # year, else use the previous year. This datetime value
83        # corresponds to the hard-coded value in the string lists
84        # below.
85        now = time.localtime()
86        # We need only month, day, hour and minute.
87        current_time_parts = now[1:5]
88        time_parts_in_listing = (12, 19, 23, 11)
89        if current_time_parts > time_parts_in_listing:
90            return now[0]
91        else:
92            return now[0] - 1
93
94    #
95    # Unix parser
96    #
97    def test_valid_unix_lines(self):
98        lines = [
99          "drwxr-sr-x   2 45854    200           512 May  4  2000 "
100            "chemeng link -> chemeng target",
101          # The year value for this line will change with the actual time.
102          "-rw-r--r--   1 45854    200          4604 Dec 19 23:11 index.html",
103          "drwxr-sr-x   2 45854    200           512 May 29  2000 os2",
104          "----------   2 45854    200           512 May 29  2000 some_file",
105          "lrwxrwxrwx   2 45854    200           512 May 29  2000 osup -> "
106                                                                  "../os2"
107        ]
108        expected_stat_results = [
109          [17901, None, None, 2, "45854", "200", 512, None,
110           (2000, 5, 4, 0, 0, 0), None, DAY_PRECISION,
111           "chemeng link", "chemeng target"],
112          [33188, None, None, 1, "45854", "200", 4604, None,
113           (self._expected_year(), 12, 19, 23, 11, 0), None, MINUTE_PRECISION,
114           "index.html", None],
115          [17901, None, None, 2, "45854", "200", 512, None,
116           (2000, 5, 29, 0, 0, 0), None, DAY_PRECISION,
117           "os2", None],
118          [32768, None, None, 2, "45854", "200", 512, None,
119           (2000, 5, 29, 0, 0, 0), None, DAY_PRECISION,
120           "some_file", None],
121          [41471, None, None, 2, "45854", "200", 512, None,
122           (2000, 5, 29, 0, 0, 0), None, DAY_PRECISION,
123           "osup", "../os2"]
124        ]
125        self._test_valid_lines(ftputil.stat.UnixParser, lines,
126                               expected_stat_results)
127
128    def test_alternative_unix_format(self):
129        # See http://ftputil.sschwarzer.net/trac/ticket/12 for a
130        # description for the need for an alternative format.
131        lines = [
132          "drwxr-sr-x   2   200           512 May  4  2000 "
133            "chemeng link -> chemeng target",
134          # The year value for this line will change with the actual time.
135          "-rw-r--r--   1   200          4604 Dec 19 23:11 index.html",
136          "drwxr-sr-x   2   200           512 May 29  2000 os2",
137          "lrwxrwxrwx   2   200           512 May 29  2000 osup -> ../os2"
138        ]
139        expected_stat_results = [
140          [17901, None, None, 2, None, "200", 512, None,
141           (2000, 5, 4, 0, 0, 0), None, DAY_PRECISION,
142           "chemeng link", "chemeng target"],
143          [33188, None, None, 1, None, "200", 4604, None,
144           (self._expected_year(), 12, 19, 23, 11, 0), None, MINUTE_PRECISION,
145           "index.html", None],
146          [17901, None, None, 2, None, "200", 512, None,
147           (2000, 5, 29, 0, 0, 0), None, DAY_PRECISION,
148           "os2", None],
149          [41471, None, None, 2, None, "200", 512, None,
150           (2000, 5, 29, 0, 0, 0), None, DAY_PRECISION,
151           "osup", "../os2"]
152        ]
153        self._test_valid_lines(ftputil.stat.UnixParser, lines,
154                               expected_stat_results)
155
156    def test_pre_epoch_times_for_unix(self):
157        # See http://ftputil.sschwarzer.net/trac/ticket/83 .
158        # `mirrors.ibiblio.org` returns dates before the "epoch" that
159        # cause an `OverflowError` in `mktime` on some platforms,
160        # e. g. Windows.
161        lines = [
162          "-rw-r--r--   1 45854    200          4604 May  4  1968 index.html",
163          "-rw-r--r--   1 45854    200          4604 Dec 31  1969 index.html",
164          "-rw-r--r--   1 45854    200          4604 May  4  1800 index.html",
165        ]
166        expected_stat_result = \
167          [33188, None, None, 1, "45854", "200", 4604, None,
168           EPOCH, None, UNKNOWN_PRECISION, "index.html", None]
169        # Make shallow copies to avoid converting the time tuple more
170        # than once in _test_valid_lines`.
171        expected_stat_results = [expected_stat_result[:],
172                                 expected_stat_result[:],
173                                 expected_stat_result[:]]
174        self._test_valid_lines(ftputil.stat.UnixParser, lines,
175                               expected_stat_results)
176
177    def test_invalid_unix_lines(self):
178        lines = [
179          # Not intended to be parsed. Should have been filtered out by
180          # `ignores_line`.
181          "total 14",
182          # Invalid month abbreviation
183          "drwxr-sr-x   2 45854    200           512 Max  4  2000 chemeng",
184          # Year value isn't an integer
185          "drwxr-sr-x   2 45854    200           512 May  4  abcd chemeng",
186          # Day value isn't an integer
187          "drwxr-sr-x   2 45854    200           512 May ab  2000 chemeng",
188          # Hour value isn't an integer
189          "-rw-r--r--   1 45854    200          4604 Dec 19 ab:11 index.html",
190          # Minute value isn't an integer
191          "-rw-r--r--   1 45854    200          4604 Dec 19 23:ab index.html",
192          # Day value too large
193          "drwxr-sr-x   2 45854    200           512 May 32  2000 chemeng",
194          # Incomplete mode
195          "drwxr-sr-    2 45854    200           512 May  4  2000 chemeng",
196          # Invalid first letter in mode
197          "xrwxr-sr-x   2 45854    200           512 May  4  2000 chemeng",
198          # Ditto, plus invalid size value
199          "xrwxr-sr-x   2 45854    200           51x May  4  2000 chemeng",
200          # Is this `os1 -> os2` pointing to `os3`, or `os1` pointing
201          # to `os2 -> os3` or the plain name `os1 -> os2 -> os3`? We
202          # don't know, so we consider the line invalid.
203          "drwxr-sr-x   2 45854    200           512 May 29  2000 "
204            "os1 -> os2 -> os3",
205          # Missing name
206          "-rwxr-sr-x   2 45854    200           51x May  4  2000 ",
207        ]
208        self._test_invalid_lines(ftputil.stat.UnixParser, lines)
209
210    #
211    # Microsoft parser
212    #
213    def test_valid_ms_lines_two_digit_year(self):
214        lines = [
215          "07-27-01  11:16AM       <DIR>          Test",
216          "10-23-95  03:25PM       <DIR>          WindowsXP",
217          "07-17-00  02:08PM             12266720 test.exe",
218          "07-17-09  12:08AM             12266720 test.exe",
219          "07-17-09  12:08PM             12266720 test.exe"
220        ]
221        expected_stat_results = [
222          [16640, None, None, None, None, None, None, None,
223           (2001, 7, 27, 11, 16, 0), None, MINUTE_PRECISION,
224           "Test", None],
225          [16640, None, None, None, None, None, None, None,
226           (1995, 10, 23, 15, 25, 0), None, MINUTE_PRECISION,
227           "WindowsXP", None],
228          [33024, None, None, None, None, None, 12266720, None,
229           (2000, 7, 17, 14, 8, 0), None, MINUTE_PRECISION,
230           "test.exe", None],
231          [33024, None, None, None, None, None, 12266720, None,
232           (2009, 7, 17, 0, 8, 0), None, MINUTE_PRECISION,
233           "test.exe", None],
234          [33024, None, None, None, None, None, 12266720, None,
235           (2009, 7, 17, 12, 8, 0), None, MINUTE_PRECISION,
236           "test.exe", None]
237        ]
238        self._test_valid_lines(ftputil.stat.MSParser, lines,
239                               expected_stat_results)
240
241    def test_valid_ms_lines_four_digit_year(self):
242        # See http://ftputil.sschwarzer.net/trac/ticket/67
243        lines = [
244          "10-19-2012  03:13PM       <DIR>          SYNCDEST",
245          "10-19-2012  03:13PM       <DIR>          SYNCSOURCE",
246          "10-19-1968  03:13PM       <DIR>          SYNC"
247        ]
248        expected_stat_results = [
249          [16640, None, None, None, None, None, None, None,
250           (2012, 10, 19, 15, 13, 0), None, MINUTE_PRECISION,
251           "SYNCDEST", None],
252          [16640, None, None, None, None, None, None, None,
253           (2012, 10, 19, 15, 13, 0), None, MINUTE_PRECISION,
254           "SYNCSOURCE", None],
255          [16640, None, None, None, None, None, None, None,
256           EPOCH, None, UNKNOWN_PRECISION,
257           "SYNC", None],
258        ]
259        self._test_valid_lines(ftputil.stat.MSParser, lines,
260                               expected_stat_results)
261
262    def test_invalid_ms_lines(self):
263        lines = [
264          # Neither "<DIR>" nor a size present
265          "07-27-01  11:16AM                      Test",
266          # "AM"/"PM" missing
267          "07-17-00  02:08             12266720 test.exe",
268          # Year not an int
269          "07-17-ab  02:08AM           12266720 test.exe",
270          # Month not an int
271          "ab-17-00  02:08AM           12266720 test.exe",
272          # Day not an int
273          "07-ab-00  02:08AM           12266720 test.exe",
274          # Hour not an int
275          "07-17-00  ab:08AM           12266720 test.exe",
276          # Invalid size value
277          "07-17-00  02:08AM           1226672x test.exe"
278        ]
279        self._test_invalid_lines(ftputil.stat.MSParser, lines)
280
281    #
282    # The following code checks if the decision logic in the Unix
283    # line parser for determining the year works.
284    #
285    def datetime_string(self, time_float):
286        """
287        Return a datetime string generated from the value in
288        `time_float`. The parameter value is a floating point value
289        as returned by `time.time()`. The returned string is built as
290        if it were from a Unix FTP server (format: MMM dd hh:mm")
291        """
292        time_tuple = time.localtime(time_float)
293        return time.strftime("%b %d %H:%M", time_tuple)
294
295    def dir_line(self, time_float):
296        """
297        Return a directory line as from a Unix FTP server. Most of
298        the contents are fixed, but the timestamp is made from
299        `time_float` (seconds since the epoch, as from `time.time()`).
300        """
301        line_template = \
302          "-rw-r--r--   1   45854   200   4604   {}   index.html"
303        return line_template.format(self.datetime_string(time_float))
304
305    def assert_equal_times(self, time1, time2):
306        """
307        Check if both times (seconds since the epoch) are equal. For
308        the purpose of this test, two times are "equal" if they
309        differ no more than one minute from each other.
310        """
311        abs_difference = abs(time1 - time2)
312        assert abs_difference <= 60.0, \
313                 "Difference is %s seconds" % abs_difference
314
315    def _test_time_shift(self, supposed_time_shift, deviation=0.0):
316        """
317        Check if the stat parser considers the time shift value
318        correctly. `deviation` is the difference between the actual
319        time shift and the supposed time shift, which is rounded
320        to full hours.
321        """
322        script = [
323          Call("__init__"),
324          Call("pwd", result="/"),
325          Call("close")
326        ]
327        with test_base.ftp_host_factory(scripted_session.factory(script)) as host:
328            host.stat_cache.disable()
329            # Explicitly use Unix format parser here.
330            host._stat._parser = ftputil.stat.UnixParser()
331            host.set_time_shift(supposed_time_shift)
332            server_time = time.time() + supposed_time_shift + deviation
333            stat_result = host._stat._parser.parse_line(self.dir_line(server_time),
334                                                        host.time_shift())
335            self.assert_equal_times(stat_result.st_mtime, server_time)
336
337    def test_time_shifts(self):
338        """Test correct year depending on time shift value."""
339        # 1. test: Client and server share the same local time
340        self._test_time_shift(0.0)
341        # 2. test: Server is three hours ahead of client
342        self._test_time_shift(3 * 60 * 60)
343        # 3. test: Client is three hours ahead of server
344        self._test_time_shift(- 3 * 60 * 60)
345        # 4. test: Server is supposed to be three hours ahead, but
346        #    is ahead three hours and one minute
347        self._test_time_shift(3 * 60 * 60, 60)
348        # 5. test: Server is supposed to be three hours ahead, but
349        #    is ahead three hours minus one minute
350        self._test_time_shift(3 * 60 * 60, -60)
351        # 6. test: Client is supposed to be three hours ahead, but
352        #    is ahead three hours and one minute
353        self._test_time_shift(-3 * 60 * 60, -60)
354        # 7. test: Client is supposed to be three hours ahead, but
355        #    is ahead three hours minus one minute
356        self._test_time_shift(-3 * 60 * 60, 60)
357
358
359class TestLstatAndStat:
360    """
361    Test `FTPHost.lstat` and `FTPHost.stat` (test currently only
362    implemented for Unix server format).
363    """
364
365    def setup_method(self, method):
366        # Most tests in this class need the mock session class with
367        # Unix format, so make this the default. Tests which need
368        # the MS format can overwrite `self.stat` later.
369        self.stat = \
370          _test_stat(session_factory=mock_ftplib.MockUnixFormatSession)
371
372    def test_repr(self):
373        """Test if the `repr` result looks like a named tuple."""
374        stat_result = self.stat._lstat("/home/sschwarzer/chemeng")
375        # TODO: Make the value for `st_mtime` robust against DST "time
376        # zone" changes.
377        expected_result = (
378          "StatResult(st_mode=17901, st_ino=None, st_dev=None, "
379          "st_nlink=2, st_uid='45854', st_gid='200', st_size=512, "
380          "st_atime=None, st_mtime=957391200.0, st_ctime=None)")
381        assert repr(stat_result) == expected_result
382
383    def test_failing_lstat(self):
384        """Test whether `lstat` fails for a nonexistent path."""
385        with pytest.raises(ftputil.error.PermanentError):
386            self.stat._lstat("/home/sschw/notthere")
387        with pytest.raises(ftputil.error.PermanentError):
388            self.stat._lstat("/home/sschwarzer/notthere")
389
390    def test_lstat_for_root(self):
391        """
392        Test `lstat` for `/` .
393
394        Note: `(l)stat` works by going one directory up and parsing
395        the output of an FTP `LIST` command. Unfortunately, it's not
396        possible to do this for the root directory `/`.
397        """
398        with pytest.raises(ftputil.error.RootDirError) as exc_info:
399            self.stat._lstat("/")
400        # `RootDirError` is "outside" the `FTPOSError` hierarchy.
401        assert not isinstance(exc_info.value, ftputil.error.FTPOSError)
402        del exc_info
403
404    def test_lstat_one_unix_file(self):
405        """Test `lstat` for a file described in Unix-style format."""
406        stat_result = self.stat._lstat("/home/sschwarzer/index.html")
407        # Second form is needed for Python 3
408        assert oct(stat_result.st_mode) in ("0100644", "0o100644")
409        assert stat_result.st_size == 4604
410        assert stat_result._st_mtime_precision == 60
411
412    def test_lstat_one_ms_file(self):
413        """Test `lstat` for a file described in DOS-style format."""
414        self.stat = _test_stat(session_factory=mock_ftplib.MockMSFormatSession)
415        stat_result = self.stat._lstat("/home/msformat/abcd.exe")
416        assert stat_result._st_mtime_precision == 60
417
418    def test_lstat_one_unix_dir(self):
419        """Test `lstat` for a directory described in Unix-style format."""
420        stat_result = self.stat._lstat("/home/sschwarzer/scios2")
421        # Second form is needed for Python 3
422        assert oct(stat_result.st_mode) in ("042755", "0o42755")
423        assert stat_result.st_ino is None
424        assert stat_result.st_dev is None
425        assert stat_result.st_nlink == 6
426        assert stat_result.st_uid == "45854"
427        assert stat_result.st_gid == "200"
428        assert stat_result.st_size == 512
429        assert stat_result.st_atime is None
430        assert (stat_result.st_mtime ==
431                stat_tuple_to_seconds((1999, 9, 20, 0, 0, 0)))
432        assert stat_result.st_ctime is None
433        assert stat_result._st_mtime_precision == 24*60*60
434        assert stat_result == (17901, None, None, 6, "45854", "200", 512, None,
435                               stat_tuple_to_seconds((1999, 9, 20, 0, 0, 0)),
436                               None)
437
438    def test_lstat_one_ms_dir(self):
439        """Test `lstat` for a directory described in DOS-style format."""
440        self.stat = _test_stat(session_factory=mock_ftplib.MockMSFormatSession)
441        stat_result = self.stat._lstat("/home/msformat/WindowsXP")
442        assert stat_result._st_mtime_precision == 60
443
444    def test_lstat_via_stat_module(self):
445        """Test `lstat` indirectly via `stat` module."""
446        stat_result = self.stat._lstat("/home/sschwarzer/")
447        assert stat.S_ISDIR(stat_result.st_mode)
448
449    def test_stat_following_link(self):
450        """Test `stat` when invoked on a link."""
451        # Simple link
452        stat_result = self.stat._stat("/home/link")
453        assert stat_result.st_size == 4604
454        # Link pointing to a link
455        stat_result = self.stat._stat("/home/python/link_link")
456        assert stat_result.st_size == 4604
457        stat_result = self.stat._stat("../python/link_link")
458        assert stat_result.st_size == 4604
459        # Recursive link structures
460        with pytest.raises(ftputil.error.PermanentError):
461            self.stat._stat("../python/bad_link")
462        with pytest.raises(ftputil.error.PermanentError):
463            self.stat._stat("/home/bad_link")
464
465    #
466    # Test automatic switching of Unix/MS parsers
467    #
468    def test_parser_switching_with_permanent_error(self):
469        """Test non-switching of parser format with `PermanentError`."""
470        self.stat = _test_stat(session_factory=mock_ftplib.MockMSFormatSession)
471        assert self.stat._allow_parser_switching is True
472        # With these directory contents, we get a `ParserError` for
473        # the Unix parser first, so `_allow_parser_switching` can be
474        # switched off no matter whether we got a `PermanentError`
475        # afterward or not.
476        with pytest.raises(ftputil.error.PermanentError):
477            self.stat._lstat("/home/msformat/nonexistent")
478        assert self.stat._allow_parser_switching is False
479
480    def test_parser_switching_default_to_unix(self):
481        """Test non-switching of parser format; stay with Unix."""
482        assert self.stat._allow_parser_switching is True
483        assert isinstance(self.stat._parser, ftputil.stat.UnixParser)
484        stat_result = self.stat._lstat("/home/sschwarzer/index.html")
485        # The Unix parser worked, so keep it.
486        assert isinstance(self.stat._parser, ftputil.stat.UnixParser)
487        assert self.stat._allow_parser_switching is False
488
489    def test_parser_switching_to_ms(self):
490        """Test switching of parser from Unix to MS format."""
491        self.stat = _test_stat(session_factory=mock_ftplib.MockMSFormatSession)
492        assert self.stat._allow_parser_switching is True
493        assert isinstance(self.stat._parser, ftputil.stat.UnixParser)
494        # Parsing the directory `/home/msformat` with the Unix parser
495        # fails, so switch to the MS parser.
496        stat_result = self.stat._lstat("/home/msformat/abcd.exe")
497        assert isinstance(self.stat._parser, ftputil.stat.MSParser)
498        assert self.stat._allow_parser_switching is False
499        assert stat_result._st_name == "abcd.exe"
500        assert stat_result.st_size == 12266720
501
502    def test_parser_switching_regarding_empty_dir(self):
503        """Test switching of parser if a directory is empty."""
504        self.stat = _test_stat(session_factory=mock_ftplib.MockMSFormatSession)
505        assert self.stat._allow_parser_switching is True
506        # When the directory we're looking into doesn't give us any
507        # lines we can't decide whether the first parser worked,
508        # because it wasn't applied. So keep the parser for now.
509        result = self.stat._listdir("/home/msformat/XPLaunch/empty")
510        assert result == []
511        assert self.stat._allow_parser_switching is True
512        assert isinstance(self.stat._parser, ftputil.stat.UnixParser)
513
514
515class TestListdir:
516    """Test `FTPHost.listdir`."""
517
518    def setup_method(self, method):
519        self.stat = \
520          _test_stat(session_factory=mock_ftplib.MockUnixFormatSession)
521
522    def test_failing_listdir(self):
523        """Test failing `FTPHost.listdir`."""
524        with pytest.raises(ftputil.error.PermanentError):
525            self.stat._listdir("notthere")
526
527    def test_succeeding_listdir(self):
528        """Test succeeding `FTPHost.listdir`."""
529        # Do we have all expected "files"?
530        assert len(self.stat._listdir(".")) == 9
531        # Have they the expected names?
532        expected = ("chemeng download image index.html os2 "
533                    "osup publications python scios2").split()
534        remote_file_list = self.stat._listdir(".")
535        for file in expected:
536            assert file in remote_file_list
Note: See TracBrowser for help on using the repository browser.