source: test/test_public_servers.py @ 1723:7d731aea5360

Last change on this file since 1723:7d731aea5360 was 1723:7d731aea5360, checked in by Stefan Schwarzer <sschwarzer@…>, 6 months ago
Remove positional argument specifiers for `format` In strings for `format` calls, remove the digits for positional arguments. For example, "{0}, {1}" becomes "{}, {}".
File size: 6.9 KB
Line 
1# Copyright (C) 2009-2018, Stefan Schwarzer <sschwarzer@sschwarzer.net>
2# and ftputil contributors (see `doc/contributors.txt`)
3# See the file LICENSE for licensing terms.
4
5import os
6import socket
7import subprocess
8
9import pytest
10
11import ftputil
12
13import test
14
15
16def email_address():
17    """
18    Return the email address used to identify the client to an
19    FTP server.
20
21    If the hostname is "warpy", use my (Stefan's) email address,
22    else try to use the content of the `$EMAIL` environment variable.
23    If that doesn't exist, use a dummy address.
24    """
25    hostname = socket.gethostname()
26    if hostname == "warpy":
27        email = "sschwarzer@sschwarzer.net"
28    else:
29        dummy_address = "anonymous@example.com"
30        email = os.environ.get("EMAIL", dummy_address)
31        if not email:
32            # Environment variable exists but content is an empty string
33            email = dummy_address
34    return email
35
36EMAIL = email_address()
37
38
39def ftp_client_listing(server, directory):
40    """
41    Log into the FTP server `server` using the command line client,
42    then change to the `directory` and retrieve a listing with "dir".
43    Return the list of items found as an `os.listdir` would return it.
44    """
45    # The `-n` option prevents an auto-login.
46    ftp_popen = subprocess.Popen(["ftp", "-n", server],
47                                 stdin=subprocess.PIPE,
48                                 stdout=subprocess.PIPE,
49                                 universal_newlines=True)
50    commands = ["user anonymous {}".format(EMAIL), "dir", "bye"]
51    if directory:
52        # Change to this directory before calling "dir".
53        commands.insert(1, "cd {}".format(directory))
54    input_ = "\n".join(commands)
55    stdout, unused_stderr = ftp_popen.communicate(input_)
56    # Collect the directory/file names from the listing's text
57    names = []
58    for line in stdout.strip().split("\n"):
59        if line.startswith("total ") or line.startswith("Trying "):
60            continue
61        parts = line.split()
62        if parts[-2] == "->":
63            # Most likely a link
64            name = parts[-3]
65        else:
66            name = parts[-1]
67        names.append(name)
68    # Remove entries for current and parent directory since they
69    # aren't included in the result of `FTPHost.listdir` either.
70    names = [name for name in names
71                  if name not in (".", "..")]
72    return names
73
74
75class TestPublicServers:
76    """
77    Get directory listings from various public FTP servers
78    with a command line client and ftputil and compare both.
79
80    An important aspect is to test different "spellings" of
81    the same directory. For example, to list the root directory
82    which is usually set after login, use "" (nothing), ".",
83    "/", "/.", "./.", "././", "..", "../.", "../.." etc.
84
85    The command line client `ftp` has to be in the path.
86    """
87
88    # Implementation note:
89    #
90    # I (Stefan) implement the code so it works with Ubuntu's
91    # client. Other clients may work or not. If you have problems
92    # testing some other client, please send me a (small) patch.
93    # Keep in mind that I don't plan supporting as many FTP
94    # obscure commandline clients as servers. ;-)
95
96    # List of pairs with server name and a directory "guaranteed
97    # to exist" under the login directory which is assumed to be
98    # the root directory.
99    servers = [# Posix format
100               ("ftp.de.debian.org", "debian"),
101               ("ftp.gnome.org",  "pub"),
102               ("ftp.heanet.ie",  "pub"),
103               ("ftp.heise.de",   "pub"),
104               # DOS/Microsoft format
105               # Do you know any FTP servers that use Microsoft
106               # format? `ftp.microsoft.com` doesn't seem to be
107               # reachable anymore.
108              ]
109
110    # This data structure contains the initial directories "." and
111    # "DIR" (which will be replaced by a valid directory name for
112    # each server). The list after the initial directory contains
113    # paths that will be queried after changing into the initial
114    # directory. All items in these lists are actually supposed to
115    # yield the same directory contents.
116    paths_table = [
117      (".", ["", ".", "/", "/.", "./.", "././", "..", "../.", "../..",
118             "DIR/..", "/DIR/../.", "/DIR/../.."]),
119      ("DIR", ["", ".", "/DIR", "/DIR/", "../DIR", "../../DIR"])
120      ]
121
122    def inner_test_server(self, server, initial_directory, paths):
123        """
124        Test one server for one initial directory.
125
126        Connect to the server `server`; if the string argument
127        `initial_directory` has a true value, change to this
128        directory. Then iterate over all strings in the sequence
129        `paths`, comparing the results of a listdir call with the
130        listing from the command line client.
131        """
132        canonical_names = ftp_client_listing(server, initial_directory)
133        host = ftputil.FTPHost(server, "anonymous", EMAIL)
134        try:
135            host.chdir(initial_directory)
136            for path in paths:
137                path = path.replace("DIR", initial_directory)
138                # Make sure that we don't recycle directory entries, i. e.
139                # really repeatedly retrieve the directory contents
140                # (shouldn't happen anyway with the current implementation).
141                host.stat_cache.clear()
142                names = host.listdir(path)
143                # Filter out "hidden" names since the FTP command line
144                # client won't include them in its listing either.
145                names = [name for name in names
146                              if not (
147                                name.startswith(".") or
148                                # The login directory of `ftp.microsoft.com`
149                                # contains this "hidden" entry that ftputil
150                                # finds but not the FTP command line client.
151                                name == "mscomtest"
152                              )]
153                failure_message = ("For server {}, directory {}: {} != {}".
154                                   format(server, initial_directory, names,
155                                          canonical_names))
156                assert names == canonical_names, failure_message
157        finally:
158            host.close()
159
160    @pytest.mark.slow_test
161    def test_servers(self):
162        """
163        Test all servers in `self.servers`.
164
165        For each server, get the listings for the login directory and
166        one other directory which is known to exist. Use different
167        "spellings" to retrieve each list via ftputil and compare with
168        the results gotten with the command line client.
169        """
170        for server, actual_initial_directory in self.servers:
171            for initial_directory, paths in self.paths_table:
172                initial_directory = initial_directory.replace(
173                                      "DIR", actual_initial_directory)
174                print(server, initial_directory)
175                self.inner_test_server(server, initial_directory, paths)
Note: See TracBrowser for help on using the repository browser.