source: test/test_stat.py @ 1666:afa733e4a84d

Last change on this file since 1666:afa733e4a84d was 1666:afa733e4a84d, checked in by Stefan Schwarzer <sschwarzer@…>, 2 years ago
Use directory entry with fixed year See [b5f33ec89ec0] for the reasoning.
File size: 23.3 KB
Line 
1# Copyright (C) 2003-2016, Stefan Schwarzer <sschwarzer@sschwarzer.net>
2# and ftputil contributors (see `doc/contributors.txt`)
3# See the file LICENSE for licensing terms.
4
5from __future__ import absolute_import
6from __future__ import division
7from __future__ import unicode_literals
8
9import stat
10import time
11
12import pytest
13
14import ftputil
15import ftputil.compat
16import ftputil.error
17import ftputil.stat
18from ftputil.stat import MINUTE_PRECISION, DAY_PRECISION, UNKNOWN_PRECISION
19
20from test import test_base
21from test import mock_ftplib
22
23
24def _test_stat(session_factory):
25    host = test_base.ftp_host_factory(session_factory=session_factory)
26    stat = ftputil.stat._Stat(host)
27    # Use Unix format parser explicitly. This doesn't exclude switching
28    # to the MS format parser later if the test allows this switching.
29    stat._parser = ftputil.stat.UnixParser()
30    return stat
31
32
33# Special value to handle special case of datetimes before the epoch.
34EPOCH = time.gmtime(0)[:6]
35
36def stat_tuple_to_seconds(t):
37    """
38    Return a float number representing the local time associated with
39    the six-element tuple `t`.
40    """
41    assert len(t) == 6, \
42             "need a six-element tuple (year, month, day, hour, min, sec)"
43    # Do _not_ apply `time.mktime` to the `EPOCH` value below. On some
44    # platforms (e. g. Windows) this might cause an `OverflowError`.
45    if t == EPOCH:
46        return 0.0
47    else:
48        return time.mktime(t + (0, 0, -1))
49
50
51class TestParsers(object):
52
53    #
54    # Helper methods
55    #
56    def _test_valid_lines(self, parser_class, lines, expected_stat_results):
57        parser = parser_class()
58        for line, expected_stat_result in zip(lines, expected_stat_results):
59            # Convert to list to compare with the list `expected_stat_results`.
60            parse_result = parser.parse_line(line)
61            stat_result = list(parse_result) + \
62                          [parse_result._st_mtime_precision,
63                           parse_result._st_name,
64                           parse_result._st_target]
65            # Convert time tuple to seconds.
66            expected_stat_result[8] = \
67              stat_tuple_to_seconds(expected_stat_result[8])
68            # Compare lists.
69            assert stat_result == expected_stat_result
70
71    def _test_invalid_lines(self, parser_class, lines):
72        parser = parser_class()
73        for line in lines:
74            with pytest.raises(ftputil.error.ParserError):
75                parser.parse_line(line)
76
77    def _expected_year(self):
78        """
79        Return the expected year for the second line in the
80        listing in `test_valid_unix_lines`.
81        """
82        # If in this year it's after Dec 19, 23:11, use the current
83        # year, else use the previous year. This datetime value
84        # corresponds to the hard-coded value in the string lists
85        # below.
86        now = time.localtime()
87        # We need only month, day, hour and minute.
88        current_time_parts = now[1:5]
89        time_parts_in_listing = (12, 19, 23, 11)
90        if current_time_parts > time_parts_in_listing:
91            return now[0]
92        else:
93            return now[0] - 1
94
95    #
96    # Unix parser
97    #
98    def test_valid_unix_lines(self):
99        lines = [
100          "drwxr-sr-x   2 45854    200           512 May  4  2000 "
101            "chemeng link -> chemeng target",
102          # The year value for this line will change with the actual time.
103          "-rw-r--r--   1 45854    200          4604 Dec 19 23:11 index.html",
104          "drwxr-sr-x   2 45854    200           512 May 29  2000 os2",
105          "----------   2 45854    200           512 May 29  2000 some_file",
106          "lrwxrwxrwx   2 45854    200           512 May 29  2000 osup -> "
107                                                                  "../os2"
108        ]
109        expected_stat_results = [
110          [17901, None, None, 2, "45854", "200", 512, None,
111           (2000, 5, 4, 0, 0, 0), None, DAY_PRECISION,
112           "chemeng link", "chemeng target"],
113          [33188, None, None, 1, "45854", "200", 4604, None,
114           (self._expected_year(), 12, 19, 23, 11, 0), None, MINUTE_PRECISION,
115           "index.html", None],
116          [17901, None, None, 2, "45854", "200", 512, None,
117           (2000, 5, 29, 0, 0, 0), None, DAY_PRECISION,
118           "os2", None],
119          [32768, None, None, 2, "45854", "200", 512, None,
120           (2000, 5, 29, 0, 0, 0), None, DAY_PRECISION,
121           "some_file", None],
122          [41471, None, None, 2, "45854", "200", 512, None,
123           (2000, 5, 29, 0, 0, 0), None, DAY_PRECISION,
124           "osup", "../os2"]
125        ]
126        self._test_valid_lines(ftputil.stat.UnixParser, lines,
127                               expected_stat_results)
128
129    def test_alternative_unix_format(self):
130        # See http://ftputil.sschwarzer.net/trac/ticket/12 for a
131        # description for the need for an alternative format.
132        lines = [
133          "drwxr-sr-x   2   200           512 May  4  2000 "
134            "chemeng link -> chemeng target",
135          # The year value for this line will change with the actual time.
136          "-rw-r--r--   1   200          4604 Dec 19 23:11 index.html",
137          "drwxr-sr-x   2   200           512 May 29  2000 os2",
138          "lrwxrwxrwx   2   200           512 May 29  2000 osup -> ../os2"
139        ]
140        expected_stat_results = [
141          [17901, None, None, 2, None, "200", 512, None,
142           (2000, 5, 4, 0, 0, 0), None, DAY_PRECISION,
143           "chemeng link", "chemeng target"],
144          [33188, None, None, 1, None, "200", 4604, None,
145           (self._expected_year(), 12, 19, 23, 11, 0), None, MINUTE_PRECISION,
146           "index.html", None],
147          [17901, None, None, 2, None, "200", 512, None,
148           (2000, 5, 29, 0, 0, 0), None, DAY_PRECISION,
149           "os2", None],
150          [41471, None, None, 2, None, "200", 512, None,
151           (2000, 5, 29, 0, 0, 0), None, DAY_PRECISION,
152           "osup", "../os2"]
153        ]
154        self._test_valid_lines(ftputil.stat.UnixParser, lines,
155                               expected_stat_results)
156
157    def test_pre_epoch_times_for_unix(self):
158        # See http://ftputil.sschwarzer.net/trac/ticket/83 .
159        # `mirrors.ibiblio.org` returns dates before the "epoch" that
160        # cause an `OverflowError` in `mktime` on some platforms,
161        # e. g. Windows.
162        lines = [
163          "-rw-r--r--   1 45854    200          4604 May  4  1968 index.html",
164          "-rw-r--r--   1 45854    200          4604 Dec 31  1969 index.html",
165          "-rw-r--r--   1 45854    200          4604 May  4  1800 index.html",
166        ]
167        expected_stat_result = \
168          [33188, None, None, 1, "45854", "200", 4604, None,
169           EPOCH, None, UNKNOWN_PRECISION, "index.html", None]
170        # Make shallow copies to avoid converting the time tuple more
171        # than once in _test_valid_lines`.
172        expected_stat_results = [expected_stat_result[:],
173                                 expected_stat_result[:],
174                                 expected_stat_result[:]]
175        self._test_valid_lines(ftputil.stat.UnixParser, lines,
176                               expected_stat_results)
177
178    def test_invalid_unix_lines(self):
179        lines = [
180          # Not intended to be parsed. Should have been filtered out by
181          # `ignores_line`.
182          "total 14",
183          # Invalid month abbreviation
184          "drwxr-sr-x   2 45854    200           512 Max  4  2000 chemeng",
185          # Year value isn't an integer
186          "drwxr-sr-x   2 45854    200           512 May  4  abcd chemeng",
187          # Day value isn't an integer
188          "drwxr-sr-x   2 45854    200           512 May ab  2000 chemeng",
189          # Hour value isn't an integer
190          "-rw-r--r--   1 45854    200          4604 Dec 19 ab:11 index.html",
191          # Minute value isn't an integer
192          "-rw-r--r--   1 45854    200          4604 Dec 19 23:ab index.html",
193          # Day value too large
194          "drwxr-sr-x   2 45854    200           512 May 32  2000 chemeng",
195          # Incomplete mode
196          "drwxr-sr-    2 45854    200           512 May  4  2000 chemeng",
197          # Invalid first letter in mode
198          "xrwxr-sr-x   2 45854    200           512 May  4  2000 chemeng",
199          # Ditto, plus invalid size value
200          "xrwxr-sr-x   2 45854    200           51x May  4  2000 chemeng",
201          # Is this `os1 -> os2` pointing to `os3`, or `os1` pointing
202          # to `os2 -> os3` or the plain name `os1 -> os2 -> os3`? We
203          # don't know, so we consider the line invalid.
204          "drwxr-sr-x   2 45854    200           512 May 29  2000 "
205            "os1 -> os2 -> os3",
206          # Missing name
207          "-rwxr-sr-x   2 45854    200           51x May  4  2000 ",
208        ]
209        self._test_invalid_lines(ftputil.stat.UnixParser, lines)
210
211    #
212    # Microsoft parser
213    #
214    def test_valid_ms_lines_two_digit_year(self):
215        lines = [
216          "07-27-01  11:16AM       <DIR>          Test",
217          "10-23-95  03:25PM       <DIR>          WindowsXP",
218          "07-17-00  02:08PM             12266720 test.exe",
219          "07-17-09  12:08AM             12266720 test.exe",
220          "07-17-09  12:08PM             12266720 test.exe"
221        ]
222        expected_stat_results = [
223          [16640, None, None, None, None, None, None, None,
224           (2001, 7, 27, 11, 16, 0), None, MINUTE_PRECISION,
225           "Test", None],
226          [16640, None, None, None, None, None, None, None,
227           (1995, 10, 23, 15, 25, 0), None, MINUTE_PRECISION,
228           "WindowsXP", None],
229          [33024, None, None, None, None, None, 12266720, None,
230           (2000, 7, 17, 14, 8, 0), None, MINUTE_PRECISION,
231           "test.exe", None],
232          [33024, None, None, None, None, None, 12266720, None,
233           (2009, 7, 17, 0, 8, 0), None, MINUTE_PRECISION,
234           "test.exe", None],
235          [33024, None, None, None, None, None, 12266720, None,
236           (2009, 7, 17, 12, 8, 0), None, MINUTE_PRECISION,
237           "test.exe", None]
238        ]
239        self._test_valid_lines(ftputil.stat.MSParser, lines,
240                               expected_stat_results)
241
242    def test_valid_ms_lines_four_digit_year(self):
243        # See http://ftputil.sschwarzer.net/trac/ticket/67
244        lines = [
245          "10-19-2012  03:13PM       <DIR>          SYNCDEST",
246          "10-19-2012  03:13PM       <DIR>          SYNCSOURCE",
247          "10-19-1968  03:13PM       <DIR>          SYNC"
248        ]
249        expected_stat_results = [
250          [16640, None, None, None, None, None, None, None,
251           (2012, 10, 19, 15, 13, 0), None, MINUTE_PRECISION,
252           "SYNCDEST", None],
253          [16640, None, None, None, None, None, None, None,
254           (2012, 10, 19, 15, 13, 0), None, MINUTE_PRECISION,
255           "SYNCSOURCE", None],
256          [16640, None, None, None, None, None, None, None,
257           EPOCH, None, UNKNOWN_PRECISION,
258           "SYNC", None],
259        ]
260        self._test_valid_lines(ftputil.stat.MSParser, lines,
261                               expected_stat_results)
262
263    def test_invalid_ms_lines(self):
264        lines = [
265          # Neither "<DIR>" nor a size present
266          "07-27-01  11:16AM                      Test",
267          # "AM"/"PM" missing
268          "07-17-00  02:08             12266720 test.exe",
269          # Year not an int
270          "07-17-ab  02:08AM           12266720 test.exe",
271          # Month not an int
272          "ab-17-00  02:08AM           12266720 test.exe",
273          # Day not an int
274          "07-ab-00  02:08AM           12266720 test.exe",
275          # Hour not an int
276          "07-17-00  ab:08AM           12266720 test.exe",
277          # Invalid size value
278          "07-17-00  02:08AM           1226672x test.exe"
279        ]
280        self._test_invalid_lines(ftputil.stat.MSParser, lines)
281
282    #
283    # The following code checks if the decision logic in the Unix
284    # line parser for determining the year works.
285    #
286    def datetime_string(self, time_float):
287        """
288        Return a datetime string generated from the value in
289        `time_float`. The parameter value is a floating point value
290        as returned by `time.time()`. The returned string is built as
291        if it were from a Unix FTP server (format: MMM dd hh:mm")
292        """
293        time_tuple = time.localtime(time_float)
294        return time.strftime("%b %d %H:%M", time_tuple)
295
296    def dir_line(self, time_float):
297        """
298        Return a directory line as from a Unix FTP server. Most of
299        the contents are fixed, but the timestamp is made from
300        `time_float` (seconds since the epoch, as from `time.time()`).
301        """
302        line_template = \
303          "-rw-r--r--   1   45854   200   4604   {0}   index.html"
304        return line_template.format(self.datetime_string(time_float))
305
306    def assert_equal_times(self, time1, time2):
307        """
308        Check if both times (seconds since the epoch) are equal. For
309        the purpose of this test, two times are "equal" if they
310        differ no more than one minute from each other.
311        """
312        abs_difference = abs(time1 - time2)
313        assert abs_difference <= 60.0, \
314                 "Difference is %s seconds" % abs_difference
315
316    def _test_time_shift(self, supposed_time_shift, deviation=0.0):
317        """
318        Check if the stat parser considers the time shift value
319        correctly. `deviation` is the difference between the actual
320        time shift and the supposed time shift, which is rounded
321        to full hours.
322        """
323        host = test_base.ftp_host_factory()
324        # Explicitly use Unix format parser here.
325        host._stat._parser = ftputil.stat.UnixParser()
326        host.set_time_shift(supposed_time_shift)
327        server_time = time.time() + supposed_time_shift + deviation
328        stat_result = host._stat._parser.parse_line(self.dir_line(server_time),
329                                                    host.time_shift())
330        self.assert_equal_times(stat_result.st_mtime, server_time)
331
332    def test_time_shifts(self):
333        """Test correct year depending on time shift value."""
334        # 1. test: Client and server share the same local time
335        self._test_time_shift(0.0)
336        # 2. test: Server is three hours ahead of client
337        self._test_time_shift(3 * 60 * 60)
338        # 3. test: Client is three hours ahead of server
339        self._test_time_shift(- 3 * 60 * 60)
340        # 4. test: Server is supposed to be three hours ahead, but
341        #    is ahead three hours and one minute
342        self._test_time_shift(3 * 60 * 60, 60)
343        # 5. test: Server is supposed to be three hours ahead, but
344        #    is ahead three hours minus one minute
345        self._test_time_shift(3 * 60 * 60, -60)
346        # 6. test: Client is supposed to be three hours ahead, but
347        #    is ahead three hours and one minute
348        self._test_time_shift(-3 * 60 * 60, -60)
349        # 7. test: Client is supposed to be three hours ahead, but
350        #    is ahead three hours minus one minute
351        self._test_time_shift(-3 * 60 * 60, 60)
352
353
354class TestLstatAndStat(object):
355    """
356    Test `FTPHost.lstat` and `FTPHost.stat` (test currently only
357    implemented for Unix server format).
358    """
359
360    def setup_method(self, method):
361        # Most tests in this class need the mock session class with
362        # Unix format, so make this the default. Tests which need
363        # the MS format can overwrite `self.stat` later.
364        self.stat = \
365          _test_stat(session_factory=mock_ftplib.MockUnixFormatSession)
366
367    def test_repr(self):
368        """Test if the `repr` result looks like a named tuple."""
369        stat_result = self.stat._lstat("/home/sschwarzer/chemeng")
370        # Only under Python 2, unicode strings have the `u` prefix.
371        # TODO: Make the value for `st_mtime` robust against DST "time
372        # zone" changes.
373        if ftputil.compat.python_version == 2:
374            expected_result = (
375              b"StatResult(st_mode=17901, st_ino=None, st_dev=None, "
376              b"st_nlink=2, st_uid=u'45854', st_gid=u'200', st_size=512, "
377              b"st_atime=None, st_mtime=957391200.0, st_ctime=None)")
378        else:
379            expected_result = (
380              "StatResult(st_mode=17901, st_ino=None, st_dev=None, "
381              "st_nlink=2, st_uid='45854', st_gid='200', st_size=512, "
382              "st_atime=None, st_mtime=957391200.0, st_ctime=None)")
383        assert repr(stat_result) == expected_result
384
385    def test_failing_lstat(self):
386        """Test whether `lstat` fails for a nonexistent path."""
387        with pytest.raises(ftputil.error.PermanentError):
388            self.stat._lstat("/home/sschw/notthere")
389        with pytest.raises(ftputil.error.PermanentError):
390            self.stat._lstat("/home/sschwarzer/notthere")
391
392    def test_lstat_for_root(self):
393        """
394        Test `lstat` for `/` .
395
396        Note: `(l)stat` works by going one directory up and parsing
397        the output of an FTP `LIST` command. Unfortunately, it's not
398        possible to do this for the root directory `/`.
399        """
400        with pytest.raises(ftputil.error.RootDirError) as exc_info:
401            self.stat._lstat("/")
402        # `RootDirError` is "outside" the `FTPOSError` hierarchy.
403        assert not isinstance(exc_info.value, ftputil.error.FTPOSError)
404        del exc_info
405
406    def test_lstat_one_unix_file(self):
407        """Test `lstat` for a file described in Unix-style format."""
408        stat_result = self.stat._lstat("/home/sschwarzer/index.html")
409        # Second form is needed for Python 3
410        assert oct(stat_result.st_mode) in ("0100644", "0o100644")
411        assert stat_result.st_size == 4604
412        assert stat_result._st_mtime_precision == 60
413
414    def test_lstat_one_ms_file(self):
415        """Test `lstat` for a file described in DOS-style format."""
416        self.stat = _test_stat(session_factory=mock_ftplib.MockMSFormatSession)
417        stat_result = self.stat._lstat("/home/msformat/abcd.exe")
418        assert stat_result._st_mtime_precision == 60
419
420    def test_lstat_one_unix_dir(self):
421        """Test `lstat` for a directory described in Unix-style format."""
422        stat_result = self.stat._lstat("/home/sschwarzer/scios2")
423        # Second form is needed for Python 3
424        assert oct(stat_result.st_mode) in ("042755", "0o42755")
425        assert stat_result.st_ino is None
426        assert stat_result.st_dev is None
427        assert stat_result.st_nlink == 6
428        assert stat_result.st_uid == "45854"
429        assert stat_result.st_gid == "200"
430        assert stat_result.st_size == 512
431        assert stat_result.st_atime is None
432        assert (stat_result.st_mtime ==
433                stat_tuple_to_seconds((1999, 9, 20, 0, 0, 0)))
434        assert stat_result.st_ctime is None
435        assert stat_result._st_mtime_precision == 24*60*60
436        assert stat_result == (17901, None, None, 6, "45854", "200", 512, None,
437                               stat_tuple_to_seconds((1999, 9, 20, 0, 0, 0)),
438                               None)
439
440    def test_lstat_one_ms_dir(self):
441        """Test `lstat` for a directory described in DOS-style format."""
442        self.stat = _test_stat(session_factory=mock_ftplib.MockMSFormatSession)
443        stat_result = self.stat._lstat("/home/msformat/WindowsXP")
444        assert stat_result._st_mtime_precision == 60
445
446    def test_lstat_via_stat_module(self):
447        """Test `lstat` indirectly via `stat` module."""
448        stat_result = self.stat._lstat("/home/sschwarzer/")
449        assert stat.S_ISDIR(stat_result.st_mode)
450
451    def test_stat_following_link(self):
452        """Test `stat` when invoked on a link."""
453        # Simple link
454        stat_result = self.stat._stat("/home/link")
455        assert stat_result.st_size == 4604
456        # Link pointing to a link
457        stat_result = self.stat._stat("/home/python/link_link")
458        assert stat_result.st_size == 4604
459        stat_result = self.stat._stat("../python/link_link")
460        assert stat_result.st_size == 4604
461        # Recursive link structures
462        with pytest.raises(ftputil.error.PermanentError):
463            self.stat._stat("../python/bad_link")
464        with pytest.raises(ftputil.error.PermanentError):
465            self.stat._stat("/home/bad_link")
466
467    #
468    # Test automatic switching of Unix/MS parsers
469    #
470    def test_parser_switching_with_permanent_error(self):
471        """Test non-switching of parser format with `PermanentError`."""
472        self.stat = _test_stat(session_factory=mock_ftplib.MockMSFormatSession)
473        assert self.stat._allow_parser_switching is True
474        # With these directory contents, we get a `ParserError` for
475        # the Unix parser first, so `_allow_parser_switching` can be
476        # switched off no matter whether we got a `PermanentError`
477        # afterward or not.
478        with pytest.raises(ftputil.error.PermanentError):
479            self.stat._lstat("/home/msformat/nonexistent")
480        assert self.stat._allow_parser_switching is False
481
482    def test_parser_switching_default_to_unix(self):
483        """Test non-switching of parser format; stay with Unix."""
484        assert self.stat._allow_parser_switching is True
485        assert isinstance(self.stat._parser, ftputil.stat.UnixParser)
486        stat_result = self.stat._lstat("/home/sschwarzer/index.html")
487        # The Unix parser worked, so keep it.
488        assert isinstance(self.stat._parser, ftputil.stat.UnixParser)
489        assert self.stat._allow_parser_switching is False
490
491    def test_parser_switching_to_ms(self):
492        """Test switching of parser from Unix to MS format."""
493        self.stat = _test_stat(session_factory=mock_ftplib.MockMSFormatSession)
494        assert self.stat._allow_parser_switching is True
495        assert isinstance(self.stat._parser, ftputil.stat.UnixParser)
496        # Parsing the directory `/home/msformat` with the Unix parser
497        # fails, so switch to the MS parser.
498        stat_result = self.stat._lstat("/home/msformat/abcd.exe")
499        assert isinstance(self.stat._parser, ftputil.stat.MSParser)
500        assert self.stat._allow_parser_switching is False
501        assert stat_result._st_name == "abcd.exe"
502        assert stat_result.st_size == 12266720
503
504    def test_parser_switching_regarding_empty_dir(self):
505        """Test switching of parser if a directory is empty."""
506        self.stat = _test_stat(session_factory=mock_ftplib.MockMSFormatSession)
507        assert self.stat._allow_parser_switching is True
508        # When the directory we're looking into doesn't give us any
509        # lines we can't decide whether the first parser worked,
510        # because it wasn't applied. So keep the parser for now.
511        result = self.stat._listdir("/home/msformat/XPLaunch/empty")
512        assert result == []
513        assert self.stat._allow_parser_switching is True
514        assert isinstance(self.stat._parser, ftputil.stat.UnixParser)
515
516
517class TestListdir(object):
518    """Test `FTPHost.listdir`."""
519
520    def setup_method(self, method):
521        self.stat = \
522          _test_stat(session_factory=mock_ftplib.MockUnixFormatSession)
523
524    def test_failing_listdir(self):
525        """Test failing `FTPHost.listdir`."""
526        with pytest.raises(ftputil.error.PermanentError):
527            self.stat._listdir("notthere")
528
529    def test_succeeding_listdir(self):
530        """Test succeeding `FTPHost.listdir`."""
531        # Do we have all expected "files"?
532        assert len(self.stat._listdir(".")) == 9
533        # Have they the expected names?
534        expected = ("chemeng download image index.html os2 "
535                    "osup publications python scios2").split()
536        remote_file_list = self.stat._listdir(".")
537        for file in expected:
538            assert file in remote_file_list
Note: See TracBrowser for help on using the repository browser.