source: test/test_stat.py @ 1660:93ea351f922b

Last change on this file since 1660:93ea351f922b was 1660:93ea351f922b, checked in by Stefan Schwarzer <sschwarzer@…>, 4 years ago
Remove `unittest.main` calls Since we now use pytest as test runner and not a shell loop in the makefile, there's no longer a need for explicit `unittest.main` calls. ticket: 98
File size: 23.3 KB
Line 
1# Copyright (C) 2003-2016, Stefan Schwarzer <sschwarzer@sschwarzer.net>
2# and ftputil contributors (see `doc/contributors.txt`)
3# See the file LICENSE for licensing terms.
4
5from __future__ import absolute_import
6from __future__ import division
7from __future__ import unicode_literals
8
9import stat
10import time
11import unittest
12
13import pytest
14
15import ftputil
16import ftputil.compat
17import ftputil.error
18import ftputil.stat
19from ftputil.stat import MINUTE_PRECISION, DAY_PRECISION, UNKNOWN_PRECISION
20
21from test import test_base
22from test import mock_ftplib
23
24
25def _test_stat(session_factory):
26    host = test_base.ftp_host_factory(session_factory=session_factory)
27    stat = ftputil.stat._Stat(host)
28    # Use Unix format parser explicitly. This doesn't exclude switching
29    # to the MS format parser later if the test allows this switching.
30    stat._parser = ftputil.stat.UnixParser()
31    return stat
32
33
34# Special value to handle special case of datetimes before the epoch.
35EPOCH = time.gmtime(0)[:6]
36
37def stat_tuple_to_seconds(t):
38    """
39    Return a float number representing the local time associated with
40    the six-element tuple `t`.
41    """
42    assert len(t) == 6, \
43             "need a six-element tuple (year, month, day, hour, min, sec)"
44    # Do _not_ apply `time.mktime` to the `EPOCH` value below. On some
45    # platforms (e. g. Windows) this might cause an `OverflowError`.
46    if t == EPOCH:
47        return 0.0
48    else:
49        return time.mktime(t + (0, 0, -1))
50
51
52class TestParsers(unittest.TestCase):
53
54    #
55    # Helper methods
56    #
57    def _test_valid_lines(self, parser_class, lines, expected_stat_results):
58        parser = parser_class()
59        for line, expected_stat_result in zip(lines, expected_stat_results):
60            # Convert to list to compare with the list `expected_stat_results`.
61            parse_result = parser.parse_line(line)
62            stat_result = list(parse_result) + \
63                          [parse_result._st_mtime_precision,
64                           parse_result._st_name,
65                           parse_result._st_target]
66            # Convert time tuple to seconds.
67            expected_stat_result[8] = \
68              stat_tuple_to_seconds(expected_stat_result[8])
69            # Compare lists.
70            assert stat_result == expected_stat_result
71
72    def _test_invalid_lines(self, parser_class, lines):
73        parser = parser_class()
74        for line in lines:
75            with pytest.raises(ftputil.error.ParserError):
76                parser.parse_line(line)
77
78    def _expected_year(self):
79        """
80        Return the expected year for the second line in the
81        listing in `test_valid_unix_lines`.
82        """
83        # If in this year it's after Dec 19, 23:11, use the current
84        # year, else use the previous year. This datetime value
85        # corresponds to the hard-coded value in the string lists
86        # below.
87        now = time.localtime()
88        # We need only month, day, hour and minute.
89        current_time_parts = now[1:5]
90        time_parts_in_listing = (12, 19, 23, 11)
91        if current_time_parts > time_parts_in_listing:
92            return now[0]
93        else:
94            return now[0] - 1
95
96    #
97    # Unix parser
98    #
99    def test_valid_unix_lines(self):
100        lines = [
101          "drwxr-sr-x   2 45854    200           512 May  4  2000 "
102            "chemeng link -> chemeng target",
103          # The year value for this line will change with the actual time.
104          "-rw-r--r--   1 45854    200          4604 Dec 19 23:11 index.html",
105          "drwxr-sr-x   2 45854    200           512 May 29  2000 os2",
106          "----------   2 45854    200           512 May 29  2000 some_file",
107          "lrwxrwxrwx   2 45854    200           512 May 29  2000 osup -> "
108                                                                  "../os2"
109        ]
110        expected_stat_results = [
111          [17901, None, None, 2, "45854", "200", 512, None,
112           (2000, 5, 4, 0, 0, 0), None, DAY_PRECISION,
113           "chemeng link", "chemeng target"],
114          [33188, None, None, 1, "45854", "200", 4604, None,
115           (self._expected_year(), 12, 19, 23, 11, 0), None, MINUTE_PRECISION,
116           "index.html", None],
117          [17901, None, None, 2, "45854", "200", 512, None,
118           (2000, 5, 29, 0, 0, 0), None, DAY_PRECISION,
119           "os2", None],
120          [32768, None, None, 2, "45854", "200", 512, None,
121           (2000, 5, 29, 0, 0, 0), None, DAY_PRECISION,
122           "some_file", None],
123          [41471, None, None, 2, "45854", "200", 512, None,
124           (2000, 5, 29, 0, 0, 0), None, DAY_PRECISION,
125           "osup", "../os2"]
126        ]
127        self._test_valid_lines(ftputil.stat.UnixParser, lines,
128                               expected_stat_results)
129
130    def test_alternative_unix_format(self):
131        # See http://ftputil.sschwarzer.net/trac/ticket/12 for a
132        # description for the need for an alternative format.
133        lines = [
134          "drwxr-sr-x   2   200           512 May  4  2000 "
135            "chemeng link -> chemeng target",
136          # The year value for this line will change with the actual time.
137          "-rw-r--r--   1   200          4604 Dec 19 23:11 index.html",
138          "drwxr-sr-x   2   200           512 May 29  2000 os2",
139          "lrwxrwxrwx   2   200           512 May 29  2000 osup -> ../os2"
140        ]
141        expected_stat_results = [
142          [17901, None, None, 2, None, "200", 512, None,
143           (2000, 5, 4, 0, 0, 0), None, DAY_PRECISION,
144           "chemeng link", "chemeng target"],
145          [33188, None, None, 1, None, "200", 4604, None,
146           (self._expected_year(), 12, 19, 23, 11, 0), None, MINUTE_PRECISION,
147           "index.html", None],
148          [17901, None, None, 2, None, "200", 512, None,
149           (2000, 5, 29, 0, 0, 0), None, DAY_PRECISION,
150           "os2", None],
151          [41471, None, None, 2, None, "200", 512, None,
152           (2000, 5, 29, 0, 0, 0), None, DAY_PRECISION,
153           "osup", "../os2"]
154        ]
155        self._test_valid_lines(ftputil.stat.UnixParser, lines,
156                               expected_stat_results)
157
158    def test_pre_epoch_times_for_unix(self):
159        # See http://ftputil.sschwarzer.net/trac/ticket/83 .
160        # `mirrors.ibiblio.org` returns dates before the "epoch" that
161        # cause an `OverflowError` in `mktime` on some platforms,
162        # e. g. Windows.
163        lines = [
164          "-rw-r--r--   1 45854    200          4604 May  4  1968 index.html",
165          "-rw-r--r--   1 45854    200          4604 Dec 31  1969 index.html",
166          "-rw-r--r--   1 45854    200          4604 May  4  1800 index.html",
167        ]
168        expected_stat_result = \
169          [33188, None, None, 1, "45854", "200", 4604, None,
170           EPOCH, None, UNKNOWN_PRECISION, "index.html", None]
171        # Make shallow copies to avoid converting the time tuple more
172        # than once in _test_valid_lines`.
173        expected_stat_results = [expected_stat_result[:],
174                                 expected_stat_result[:],
175                                 expected_stat_result[:]]
176        self._test_valid_lines(ftputil.stat.UnixParser, lines,
177                               expected_stat_results)
178
179    def test_invalid_unix_lines(self):
180        lines = [
181          # Not intended to be parsed. Should have been filtered out by
182          # `ignores_line`.
183          "total 14",
184          # Invalid month abbreviation
185          "drwxr-sr-x   2 45854    200           512 Max  4  2000 chemeng",
186          # Year value isn't an integer
187          "drwxr-sr-x   2 45854    200           512 May  4  abcd chemeng",
188          # Day value isn't an integer
189          "drwxr-sr-x   2 45854    200           512 May ab  2000 chemeng",
190          # Hour value isn't an integer
191          "-rw-r--r--   1 45854    200          4604 Dec 19 ab:11 index.html",
192          # Minute value isn't an integer
193          "-rw-r--r--   1 45854    200          4604 Dec 19 23:ab index.html",
194          # Day value too large
195          "drwxr-sr-x   2 45854    200           512 May 32  2000 chemeng",
196          # Incomplete mode
197          "drwxr-sr-    2 45854    200           512 May  4  2000 chemeng",
198          # Invalid first letter in mode
199          "xrwxr-sr-x   2 45854    200           512 May  4  2000 chemeng",
200          # Ditto, plus invalid size value
201          "xrwxr-sr-x   2 45854    200           51x May  4  2000 chemeng",
202          # Is this `os1 -> os2` pointing to `os3`, or `os1` pointing
203          # to `os2 -> os3` or the plain name `os1 -> os2 -> os3`? We
204          # don't know, so we consider the line invalid.
205          "drwxr-sr-x   2 45854    200           512 May 29  2000 "
206            "os1 -> os2 -> os3",
207          # Missing name
208          "-rwxr-sr-x   2 45854    200           51x May  4  2000 ",
209        ]
210        self._test_invalid_lines(ftputil.stat.UnixParser, lines)
211
212    #
213    # Microsoft parser
214    #
215    def test_valid_ms_lines_two_digit_year(self):
216        lines = [
217          "07-27-01  11:16AM       <DIR>          Test",
218          "10-23-95  03:25PM       <DIR>          WindowsXP",
219          "07-17-00  02:08PM             12266720 test.exe",
220          "07-17-09  12:08AM             12266720 test.exe",
221          "07-17-09  12:08PM             12266720 test.exe"
222        ]
223        expected_stat_results = [
224          [16640, None, None, None, None, None, None, None,
225           (2001, 7, 27, 11, 16, 0), None, MINUTE_PRECISION,
226           "Test", None],
227          [16640, None, None, None, None, None, None, None,
228           (1995, 10, 23, 15, 25, 0), None, MINUTE_PRECISION,
229           "WindowsXP", None],
230          [33024, None, None, None, None, None, 12266720, None,
231           (2000, 7, 17, 14, 8, 0), None, MINUTE_PRECISION,
232           "test.exe", None],
233          [33024, None, None, None, None, None, 12266720, None,
234           (2009, 7, 17, 0, 8, 0), None, MINUTE_PRECISION,
235           "test.exe", None],
236          [33024, None, None, None, None, None, 12266720, None,
237           (2009, 7, 17, 12, 8, 0), None, MINUTE_PRECISION,
238           "test.exe", None]
239        ]
240        self._test_valid_lines(ftputil.stat.MSParser, lines,
241                               expected_stat_results)
242
243    def test_valid_ms_lines_four_digit_year(self):
244        # See http://ftputil.sschwarzer.net/trac/ticket/67
245        lines = [
246          "10-19-2012  03:13PM       <DIR>          SYNCDEST",
247          "10-19-2012  03:13PM       <DIR>          SYNCSOURCE",
248          "10-19-1968  03:13PM       <DIR>          SYNC"
249        ]
250        expected_stat_results = [
251          [16640, None, None, None, None, None, None, None,
252           (2012, 10, 19, 15, 13, 0), None, MINUTE_PRECISION,
253           "SYNCDEST", None],
254          [16640, None, None, None, None, None, None, None,
255           (2012, 10, 19, 15, 13, 0), None, MINUTE_PRECISION,
256           "SYNCSOURCE", None],
257          [16640, None, None, None, None, None, None, None,
258           EPOCH, None, UNKNOWN_PRECISION,
259           "SYNC", None],
260        ]
261        self._test_valid_lines(ftputil.stat.MSParser, lines,
262                               expected_stat_results)
263
264    def test_invalid_ms_lines(self):
265        lines = [
266          # Neither "<DIR>" nor a size present
267          "07-27-01  11:16AM                      Test",
268          # "AM"/"PM" missing
269          "07-17-00  02:08             12266720 test.exe",
270          # Year not an int
271          "07-17-ab  02:08AM           12266720 test.exe",
272          # Month not an int
273          "ab-17-00  02:08AM           12266720 test.exe",
274          # Day not an int
275          "07-ab-00  02:08AM           12266720 test.exe",
276          # Hour not an int
277          "07-17-00  ab:08AM           12266720 test.exe",
278          # Invalid size value
279          "07-17-00  02:08AM           1226672x test.exe"
280        ]
281        self._test_invalid_lines(ftputil.stat.MSParser, lines)
282
283    #
284    # The following code checks if the decision logic in the Unix
285    # line parser for determining the year works.
286    #
287    def datetime_string(self, time_float):
288        """
289        Return a datetime string generated from the value in
290        `time_float`. The parameter value is a floating point value
291        as returned by `time.time()`. The returned string is built as
292        if it were from a Unix FTP server (format: MMM dd hh:mm")
293        """
294        time_tuple = time.localtime(time_float)
295        return time.strftime("%b %d %H:%M", time_tuple)
296
297    def dir_line(self, time_float):
298        """
299        Return a directory line as from a Unix FTP server. Most of
300        the contents are fixed, but the timestamp is made from
301        `time_float` (seconds since the epoch, as from `time.time()`).
302        """
303        line_template = \
304          "-rw-r--r--   1   45854   200   4604   {0}   index.html"
305        return line_template.format(self.datetime_string(time_float))
306
307    def assert_equal_times(self, time1, time2):
308        """
309        Check if both times (seconds since the epoch) are equal. For
310        the purpose of this test, two times are "equal" if they
311        differ no more than one minute from each other.
312        """
313        abs_difference = abs(time1 - time2)
314        assert abs_difference <= 60.0, \
315                 "Difference is %s seconds" % abs_difference
316
317    def _test_time_shift(self, supposed_time_shift, deviation=0.0):
318        """
319        Check if the stat parser considers the time shift value
320        correctly. `deviation` is the difference between the actual
321        time shift and the supposed time shift, which is rounded
322        to full hours.
323        """
324        host = test_base.ftp_host_factory()
325        # Explicitly use Unix format parser here.
326        host._stat._parser = ftputil.stat.UnixParser()
327        host.set_time_shift(supposed_time_shift)
328        server_time = time.time() + supposed_time_shift + deviation
329        stat_result = host._stat._parser.parse_line(self.dir_line(server_time),
330                                                    host.time_shift())
331        self.assert_equal_times(stat_result.st_mtime, server_time)
332
333    def test_time_shifts(self):
334        """Test correct year depending on time shift value."""
335        # 1. test: Client and server share the same local time
336        self._test_time_shift(0.0)
337        # 2. test: Server is three hours ahead of client
338        self._test_time_shift(3 * 60 * 60)
339        # 3. test: Client is three hours ahead of server
340        self._test_time_shift(- 3 * 60 * 60)
341        # 4. test: Server is supposed to be three hours ahead, but
342        #    is ahead three hours and one minute
343        self._test_time_shift(3 * 60 * 60, 60)
344        # 5. test: Server is supposed to be three hours ahead, but
345        #    is ahead three hours minus one minute
346        self._test_time_shift(3 * 60 * 60, -60)
347        # 6. test: Client is supposed to be three hours ahead, but
348        #    is ahead three hours and one minute
349        self._test_time_shift(-3 * 60 * 60, -60)
350        # 7. test: Client is supposed to be three hours ahead, but
351        #    is ahead three hours minus one minute
352        self._test_time_shift(-3 * 60 * 60, 60)
353
354
355class TestLstatAndStat(unittest.TestCase):
356    """
357    Test `FTPHost.lstat` and `FTPHost.stat` (test currently only
358    implemented for Unix server format).
359    """
360
361    def setUp(self):
362        # Most tests in this class need the mock session class with
363        # Unix format, so make this the default. Tests which need
364        # the MS format can overwrite `self.stat` later.
365        self.stat = \
366          _test_stat(session_factory=mock_ftplib.MockUnixFormatSession)
367
368    def test_repr(self):
369        """Test if the `repr` result looks like a named tuple."""
370        stat_result = self.stat._lstat("/home/sschwarzer/index.html")
371        # Only under Python 2, unicode strings have the `u` prefix.
372        # TODO: Make the value for `st_mtime` robust against DST "time
373        # zone" changes.
374        if ftputil.compat.python_version == 2:
375            expected_result = (
376              b"StatResult(st_mode=33188, st_ino=None, st_dev=None, "
377              b"st_nlink=1, st_uid=u'45854', st_gid=u'200', st_size=4604, "
378              b"st_atime=None, st_mtime=1453241460.0, st_ctime=None)")
379        else:
380            expected_result = (
381              "StatResult(st_mode=33188, st_ino=None, st_dev=None, "
382              "st_nlink=1, st_uid='45854', st_gid='200', st_size=4604, "
383              "st_atime=None, st_mtime=1453241460.0, st_ctime=None)")
384        assert repr(stat_result) == expected_result
385
386    def test_failing_lstat(self):
387        """Test whether `lstat` fails for a nonexistent path."""
388        with pytest.raises(ftputil.error.PermanentError):
389            self.stat._lstat("/home/sschw/notthere")
390        with pytest.raises(ftputil.error.PermanentError):
391            self.stat._lstat("/home/sschwarzer/notthere")
392
393    def test_lstat_for_root(self):
394        """
395        Test `lstat` for `/` .
396
397        Note: `(l)stat` works by going one directory up and parsing
398        the output of an FTP `LIST` command. Unfortunately, it's not
399        possible to do this for the root directory `/`.
400        """
401        with pytest.raises(ftputil.error.RootDirError) as exc_info:
402            self.stat._lstat("/")
403        # `RootDirError` is "outside" the `FTPOSError` hierarchy.
404        assert not isinstance(exc_info.value, ftputil.error.FTPOSError)
405        del exc_info
406
407    def test_lstat_one_unix_file(self):
408        """Test `lstat` for a file described in Unix-style format."""
409        stat_result = self.stat._lstat("/home/sschwarzer/index.html")
410        # Second form is needed for Python 3
411        assert oct(stat_result.st_mode) in ("0100644", "0o100644")
412        assert stat_result.st_size == 4604
413        assert stat_result._st_mtime_precision == 60
414
415    def test_lstat_one_ms_file(self):
416        """Test `lstat` for a file described in DOS-style format."""
417        self.stat = _test_stat(session_factory=mock_ftplib.MockMSFormatSession)
418        stat_result = self.stat._lstat("/home/msformat/abcd.exe")
419        assert stat_result._st_mtime_precision == 60
420
421    def test_lstat_one_unix_dir(self):
422        """Test `lstat` for a directory described in Unix-style format."""
423        stat_result = self.stat._lstat("/home/sschwarzer/scios2")
424        # Second form is needed for Python 3
425        assert oct(stat_result.st_mode) in ("042755", "0o42755")
426        assert stat_result.st_ino is None
427        assert stat_result.st_dev is None
428        assert stat_result.st_nlink == 6
429        assert stat_result.st_uid == "45854"
430        assert stat_result.st_gid == "200"
431        assert stat_result.st_size == 512
432        assert stat_result.st_atime is None
433        assert (stat_result.st_mtime ==
434                stat_tuple_to_seconds((1999, 9, 20, 0, 0, 0)))
435        assert stat_result.st_ctime is None
436        assert stat_result._st_mtime_precision == 24*60*60
437        assert stat_result == (17901, None, None, 6, "45854", "200", 512, None,
438                               stat_tuple_to_seconds((1999, 9, 20, 0, 0, 0)),
439                               None)
440
441    def test_lstat_one_ms_dir(self):
442        """Test `lstat` for a directory described in DOS-style format."""
443        self.stat = _test_stat(session_factory=mock_ftplib.MockMSFormatSession)
444        stat_result = self.stat._lstat("/home/msformat/WindowsXP")
445        assert stat_result._st_mtime_precision == 60
446
447    def test_lstat_via_stat_module(self):
448        """Test `lstat` indirectly via `stat` module."""
449        stat_result = self.stat._lstat("/home/sschwarzer/")
450        assert stat.S_ISDIR(stat_result.st_mode)
451
452    def test_stat_following_link(self):
453        """Test `stat` when invoked on a link."""
454        # Simple link
455        stat_result = self.stat._stat("/home/link")
456        assert stat_result.st_size == 4604
457        # Link pointing to a link
458        stat_result = self.stat._stat("/home/python/link_link")
459        assert stat_result.st_size == 4604
460        stat_result = self.stat._stat("../python/link_link")
461        assert stat_result.st_size == 4604
462        # Recursive link structures
463        with pytest.raises(ftputil.error.PermanentError):
464            self.stat._stat("../python/bad_link")
465        with pytest.raises(ftputil.error.PermanentError):
466            self.stat._stat("/home/bad_link")
467
468    #
469    # Test automatic switching of Unix/MS parsers
470    #
471    def test_parser_switching_with_permanent_error(self):
472        """Test non-switching of parser format with `PermanentError`."""
473        self.stat = _test_stat(session_factory=mock_ftplib.MockMSFormatSession)
474        assert self.stat._allow_parser_switching is True
475        # With these directory contents, we get a `ParserError` for
476        # the Unix parser first, so `_allow_parser_switching` can be
477        # switched off no matter whether we got a `PermanentError`
478        # afterward or not.
479        with pytest.raises(ftputil.error.PermanentError):
480            self.stat._lstat("/home/msformat/nonexistent")
481        assert self.stat._allow_parser_switching is False
482
483    def test_parser_switching_default_to_unix(self):
484        """Test non-switching of parser format; stay with Unix."""
485        assert self.stat._allow_parser_switching is True
486        assert isinstance(self.stat._parser, ftputil.stat.UnixParser)
487        stat_result = self.stat._lstat("/home/sschwarzer/index.html")
488        # The Unix parser worked, so keep it.
489        assert isinstance(self.stat._parser, ftputil.stat.UnixParser)
490        assert self.stat._allow_parser_switching is False
491
492    def test_parser_switching_to_ms(self):
493        """Test switching of parser from Unix to MS format."""
494        self.stat = _test_stat(session_factory=mock_ftplib.MockMSFormatSession)
495        assert self.stat._allow_parser_switching is True
496        assert isinstance(self.stat._parser, ftputil.stat.UnixParser)
497        # Parsing the directory `/home/msformat` with the Unix parser
498        # fails, so switch to the MS parser.
499        stat_result = self.stat._lstat("/home/msformat/abcd.exe")
500        assert isinstance(self.stat._parser, ftputil.stat.MSParser)
501        assert self.stat._allow_parser_switching is False
502        assert stat_result._st_name == "abcd.exe"
503        assert stat_result.st_size == 12266720
504
505    def test_parser_switching_regarding_empty_dir(self):
506        """Test switching of parser if a directory is empty."""
507        self.stat = _test_stat(session_factory=mock_ftplib.MockMSFormatSession)
508        assert self.stat._allow_parser_switching is True
509        # When the directory we're looking into doesn't give us any
510        # lines we can't decide whether the first parser worked,
511        # because it wasn't applied. So keep the parser for now.
512        result = self.stat._listdir("/home/msformat/XPLaunch/empty")
513        assert result == []
514        assert self.stat._allow_parser_switching is True
515        assert isinstance(self.stat._parser, ftputil.stat.UnixParser)
516
517
518class TestListdir(unittest.TestCase):
519    """Test `FTPHost.listdir`."""
520
521    def setUp(self):
522        self.stat = \
523          _test_stat(session_factory=mock_ftplib.MockUnixFormatSession)
524
525    def test_failing_listdir(self):
526        """Test failing `FTPHost.listdir`."""
527        with pytest.raises(ftputil.error.PermanentError):
528            self.stat._listdir("notthere")
529
530    def test_succeeding_listdir(self):
531        """Test succeeding `FTPHost.listdir`."""
532        # Do we have all expected "files"?
533        assert len(self.stat._listdir(".")) == 9
534        # Have they the expected names?
535        expected = ("chemeng download image index.html os2 "
536                    "osup publications python scios2").split()
537        remote_file_list = self.stat._listdir(".")
538        for file in expected:
539            assert file in remote_file_list
Note: See TracBrowser for help on using the repository browser.