source: test/test_public_servers.py @ 1712:1b17d07f3a88

Last change on this file since 1712:1b17d07f3a88 was 1699:906d7eda1e0b, checked in by Stefan Schwarzer <sschwarzer@…>, 23 months ago
Add remark on "missing" ftp.microsoft.com
File size: 7.0 KB
Line 
1# Copyright (C) 2009-2016, Stefan Schwarzer <sschwarzer@sschwarzer.net>
2# and ftputil contributors (see `doc/contributors.txt`)
3# See the file LICENSE for licensing terms.
4
5from __future__ import print_function
6from __future__ import unicode_literals
7
8import os
9import socket
10import subprocess
11
12import pytest
13
14import ftputil
15
16import test
17
18
19def email_address():
20    """
21    Return the email address used to identify the client to an
22    FTP server.
23
24    If the hostname is "warpy", use my (Stefan's) email address,
25    else try to use the content of the `$EMAIL` environment variable.
26    If that doesn't exist, use a dummy address.
27    """
28    hostname = socket.gethostname()
29    if hostname == "warpy":
30        email = "sschwarzer@sschwarzer.net"
31    else:
32        dummy_address = "anonymous@example.com"
33        email = os.environ.get("EMAIL", dummy_address)
34        if not email:
35            # Environment variable exists but content is an empty string
36            email = dummy_address
37    return email
38
39EMAIL = email_address()
40
41
42def ftp_client_listing(server, directory):
43    """
44    Log into the FTP server `server` using the command line client,
45    then change to the `directory` and retrieve a listing with "dir".
46    Return the list of items found as an `os.listdir` would return it.
47    """
48    # The `-n` option prevents an auto-login.
49    ftp_popen = subprocess.Popen(["ftp", "-n", server],
50                                 stdin=subprocess.PIPE,
51                                 stdout=subprocess.PIPE,
52                                 universal_newlines=True)
53    commands = ["user anonymous {0}".format(EMAIL), "dir", "bye"]
54    if directory:
55        # Change to this directory before calling "dir".
56        commands.insert(1, "cd {0}".format(directory))
57    input_ = "\n".join(commands)
58    stdout, unused_stderr = ftp_popen.communicate(input_)
59    # Collect the directory/file names from the listing's text
60    names = []
61    for line in stdout.strip().split("\n"):
62        if line.startswith("total ") or line.startswith("Trying "):
63            continue
64        parts = line.split()
65        if parts[-2] == "->":
66            # Most likely a link
67            name = parts[-3]
68        else:
69            name = parts[-1]
70        names.append(name)
71    # Remove entries for current and parent directory since they
72    # aren't included in the result of `FTPHost.listdir` either.
73    names = [name for name in names
74                  if name not in (".", "..")]
75    return names
76
77
78class TestPublicServers(object):
79    """
80    Get directory listings from various public FTP servers
81    with a command line client and ftputil and compare both.
82
83    An important aspect is to test different "spellings" of
84    the same directory. For example, to list the root directory
85    which is usually set after login, use "" (nothing), ".",
86    "/", "/.", "./.", "././", "..", "../.", "../.." etc.
87
88    The command line client `ftp` has to be in the path.
89    """
90
91    # Implementation note:
92    #
93    # I (Stefan) implement the code so it works with Ubuntu's
94    # client. Other clients may work or not. If you have problems
95    # testing some other client, please send me a (small) patch.
96    # Keep in mind that I don't plan supporting as many FTP
97    # obscure commandline clients as servers. ;-)
98
99    # List of pairs with server name and a directory "guaranteed
100    # to exist" under the login directory which is assumed to be
101    # the root directory.
102    servers = [# Posix format
103               ("ftp.de.debian.org", "debian"),
104               ("ftp.gnome.org",  "pub"),
105               ("ftp.heanet.ie",  "pub"),
106               ("ftp.heise.de",   "pub"),
107               # DOS/Microsoft format
108               # Do you know any FTP servers that use Microsoft
109               # format? `ftp.microsoft.com` doesn't seem to be
110               # reachable anymore.
111              ]
112
113    # This data structure contains the initial directories "." and
114    # "DIR" (which will be replaced by a valid directory name for
115    # each server). The list after the initial directory contains
116    # paths that will be queried after changing into the initial
117    # directory. All items in these lists are actually supposed to
118    # yield the same directory contents.
119    paths_table = [
120      (".", ["", ".", "/", "/.", "./.", "././", "..", "../.", "../..",
121             "DIR/..", "/DIR/../.", "/DIR/../.."]),
122      ("DIR", ["", ".", "/DIR", "/DIR/", "../DIR", "../../DIR"])
123      ]
124
125    def inner_test_server(self, server, initial_directory, paths):
126        """
127        Test one server for one initial directory.
128
129        Connect to the server `server`; if the string argument
130        `initial_directory` has a true value, change to this
131        directory. Then iterate over all strings in the sequence
132        `paths`, comparing the results of a listdir call with the
133        listing from the command line client.
134        """
135        canonical_names = ftp_client_listing(server, initial_directory)
136        host = ftputil.FTPHost(server, "anonymous", EMAIL)
137        try:
138            host.chdir(initial_directory)
139            for path in paths:
140                path = path.replace("DIR", initial_directory)
141                # Make sure that we don't recycle directory entries, i. e.
142                # really repeatedly retrieve the directory contents
143                # (shouldn't happen anyway with the current implementation).
144                host.stat_cache.clear()
145                names = host.listdir(path)
146                # Filter out "hidden" names since the FTP command line
147                # client won't include them in its listing either.
148                names = [name for name in names
149                              if not (
150                                name.startswith(".") or
151                                # The login directory of `ftp.microsoft.com`
152                                # contains this "hidden" entry that ftputil
153                                # finds but not the FTP command line client.
154                                name == "mscomtest"
155                              )]
156                failure_message = ("For server {0}, directory {1}: {2} != {3}".
157                                   format(server, initial_directory, names,
158                                          canonical_names))
159                assert names == canonical_names, failure_message
160        finally:
161            host.close()
162
163    @pytest.mark.slow_test
164    def test_servers(self):
165        """
166        Test all servers in `self.servers`.
167
168        For each server, get the listings for the login directory and
169        one other directory which is known to exist. Use different
170        "spellings" to retrieve each list via ftputil and compare with
171        the results gotten with the command line client.
172        """
173        for server, actual_initial_directory in self.servers:
174            for initial_directory, paths in self.paths_table:
175                initial_directory = initial_directory.replace(
176                                      "DIR", actual_initial_directory)
177                print(server, initial_directory)
178                self.inner_test_server(server, initial_directory, paths)
Note: See TracBrowser for help on using the repository browser.