Index: hgtags =================================================================== --- .hgtags (revision 833:b13b95ef169b) +++ (revision ) @@ -1,34 +1,0 @@ -003a40a10b6df8754bb9a3f9bd519315963923f2 release1_1_4 -04ab5a54489bed14c7f7a4e85d2643a7054ffbf6 release1_1_1 -166654a407e89da9cfdcf4bb24875a3b9d9f9d8d release1_1_3 -17ce2164b30f342f89dac09ef6245054f27d199b Refactoring_2006-02-04 -1c2921b9f36c278c8530a45e3c7b2a3ee7261c58 release2_3 -210e88e14aff7b275641b0452c4de8f70020d47a release2_1b -2d03f6315e4508eb22c242a09400bfe1b9b5cf1e release2_0_b1 -2e7adfc91a3c1907ab4023d4b206aae934f9895e release2_0_3b -2f44a5efbf43e7a185fc3fffecd2b9ecfaa11ef0 release2_1b2 -3335b387c7d239f8390913dd4636c70ad9fd1b3a release2_4_2 -3a8af43ad6f5355ec19317e4d8945af2b3f8cf20 release2_0_3b2 -4683d9ed51ea894b89708b27bf5b7bb35f92ac83 release2_2a1 -5030c7856184162115795d290fec8fff6da3469d release2_4b -6f76e6fd41ab17cc9086c4e8de1dadde365830c2 release2_4_2b -709496c34c87e96e48b4b1ac6df9a178025a840c release2_1 -81a5e60f62ab15d55b41c1b039ab5cfdbc843c51 release2_0_b3 -85b8c11b21829fb99e2472bba2440d956c7a8027 release1_1 -8e3fbb31cc805ea4323de49e6bcc8e7c8b30cbbd release2_2 -99e6d4a2c5a0f52e27d37de41d0d308e074d7828 release1_1_b2 -a09e4a4f0c9d3025c2beab5f40a1533a53f06d00 release2_2_3 -a9fe73654ee15cb5d40e3c72289ac8e065726440 Refactoring_2003-03-16 -aa47ecfa21a6a57e4d1d98bb8e6972bcb6a19aa8 release2_0 -aaeff6db25783dc857d40adc3ce4401b371f29b6 release2_4_1 -c1ba407a251de7e68631b3392c180f11e065d1ac release2_0_1 -c223441a5d31b638d719178023cc66333db0bc35 release2_4_2b2 -d42a78d1f5607cbc17264eaa3eb21885e991bdfe release2_0_2 -e555c14cb2f2b3ff3ef4c1d4677fe35118e9bc26 release2_2_2 -e940c4c4a23b0279a3d2012aafd7ee702c57365f release2_2b2 -ea898f19e4cebde5172fab67d5bfb96e9ce3f9aa release2_1_1 -f8d0c19e1722a3e16eda9e21d305340d2aecabc3 release2_0_3 -fbc868918cf7e3c2e0983c58502b6b7bd59adabf release2_2_4 -fc1a1df64b691672233e5f12fdc3cd54c03fd722 release1_1_b1 -fee84e3496a65622608f1e362fa8afaf730e979c release2_2b -ffeb6a4e7b534d6abd594d121489e862333172e1 release1_1_2 Index: MANIFEST =================================================================== --- MANIFEST (revision 793:0ef45c839810) +++ MANIFEST (revision 575:2654b5037ac0) @@ -1,31 +1,23 @@ -default.css -find_deprecated_code.py +MANIFEST +README.html +README.txt +VERSION +__init__.py +_mock_ftplib.py +_test_base.py +_test_ftp_path.py +_test_ftp_stat.py +_test_ftp_stat_cache.py +_test_ftputil.py ftp_error.py ftp_file.py ftp_path.py +ftp_stat.py ftp_stat_cache.py -ftp_stat.py ftputil.html ftputil.py +ftputil.txt ftputil_ru_utf8.txt -ftputil.txt ftputil_version.py -__init__.py lrucache.py -MANIFEST -_mock_ftplib.py -PKG-INFO -README.html -README.txt setup.py -_test_base.py -_test_ftp_error.py -_test_ftp_file.py -_test_ftp_path.py -_test_ftp_stat_cache.py -_test_ftp_stat.py -_test_ftputil.py -_test_python2_4.py -_test_real_ftp.py -_test_with_statement.py -VERSION Index: Makefile =================================================================== --- Makefile (revision 816:2a804a48e36d) +++ Makefile (revision 537:468cd27a4377) @@ -1,3 +1,3 @@ -# Copyright (C) 2003-2008, Stefan Schwarzer +# Copyright (C) 2003-2006, Stefan Schwarzer # All rights reserved. # @@ -35,42 +35,25 @@ SHELL=/bin/sh PROJECT_DIR=/home/schwa/sd/python/ftputil -VERSION=$(shell cat VERSION) -DEBIAN_DIR=${PROJECT_DIR}/debian DOC_FILES=README.html ftputil.html ftputil_ru.html -TMP_LS_FILE=tmp_ls.out -STYLESHEET_PATH=default.css +STYLESHEET_PATH=/usr/share/doc/docutils-0.3.7/html/tools/stylesheets/default.css WWW_DIR=${HOME}/www SED=sed -i'' -r -e -RST2HTML=rst2html -PRODUCTION_FILES=ftp_error.py ftp_file.py ftp_path.py ftp_stat_cache.py \ - ftp_stat.py ftputil.py ftputil_version.py __init__.py \ - find_deprecated_code.py -# name test files; make sure the long-running tests come last -TEST_FILES=$(shell ls _test_*.py | \ - sed -e "s/_test_real_ftp.py//" | \ - sed -e "s/_test_public_servers.py//" ) \ - _test_real_ftp.py _test_public_servers.py -.PHONY: dist extdist test pylint docs clean register patch debdistclean debdist +.PHONY: dist extdist test docs clean register patch .SUFFIXES: .txt .html test: - @echo "Tests for ftputil ${VERSION}\n" - python2.4 _test_python2_4.py - for file in $(TEST_FILES); \ + for file in `ls _test_*.py`; \ do \ - echo $$file ; \ python $$file ; \ done -pylint: - pylint --rcfile=pylintrc ${PRODUCTION_FILES} | less ftputil_ru.html: ftputil_ru_utf8.txt - ${RST2HTML} --stylesheet-path=${STYLESHEET_PATH} --embed-stylesheet \ + rst2html.py --stylesheet-path=${STYLESHEET_PATH} --embed-stylesheet \ --input-encoding=utf-8 $< $@ .txt.html: - ${RST2HTML} --stylesheet-path=${STYLESHEET_PATH} --embed-stylesheet $< $@ + rst2html.py --stylesheet-path=${STYLESHEET_PATH} --embed-stylesheet $< $@ patch: @@ -78,44 +61,19 @@ ${SED} "s/^__version__ = '.*'/__version__ = \'`cat VERSION`\'/" \ ftputil_version.py - ${SED} "s/^:Version: .*/:Version: ${VERSION}/" ftputil.txt + ${SED} "s/^:Version: .*/:Version: `cat VERSION`/" ftputil.txt ${SED} "s/^:Date: .*/:Date: `date +"%Y-%m-%d"`/" ftputil.txt #TODO add rules for Russian translation - ${SED} "s/^Version: .*/Version: ${VERSION}/" PKG-INFO - ${SED} "s/(\/wiki\/Download\/ftputil-).*(\.tar\.gz)/\1${VERSION}\2/" \ + ${SED} "s/^Version: .*/Version: `cat VERSION`/" PKG-INFO + ${SED} "s/(\/wiki\/Download\/ftputil-).*(\.tar\.gz)/\1`cat VERSION`\2/" \ PKG-INFO docs: ${DOC_FILES} README.txt ftputil.txt ftputil_ru_utf8.txt -manifestdiff: MANIFEST - @ls -1 | grep -v .pyc | grep -v ${TMP_LS_FILE} > ${TMP_LS_FILE} - -diff -u MANIFEST ${TMP_LS_FILE} - @rm ${TMP_LS_FILE} - -dist: clean patch test pylint docs +dist: clean patch docs python setup.py sdist - -debdistclean: - cd ${DEBIAN_DIR} && rm -rf `ls -1 | grep -v "^custom$$"` - -debdist: debdistclean - cp dist/ftputil-${VERSION}.tar.gz \ - ${DEBIAN_DIR}/ftputil-${VERSION}.orig.tar.gz - tar -x -C ${DEBIAN_DIR} -zf ${DEBIAN_DIR}/ftputil-${VERSION}.orig.tar.gz - cd ${DEBIAN_DIR}/ftputil-${VERSION} && \ - echo "\n" | dh_make --copyright bsd --single --cdbs && \ - cd debian && \ - rm *.ex *.EX dirs README.Debian - # copy custom files (control, rules, copyright, changelog, maybe others) - cp ${DEBIAN_DIR}/custom/* ${DEBIAN_DIR}/ftputil-${VERSION}/debian - cd ${DEBIAN_DIR}/ftputil-${VERSION} && \ - dpkg-buildpackage -us -uc - # put the Debian package beneath the .tar.gz files - cp ${DEBIAN_DIR}/python-ftputil_${VERSION}-?_all.deb dist - # final check (better than nothing) - lintian ${DEBIAN_DIR}/python-ftputil_${VERSION}-?_all.deb localcopy: @echo "Copying archive and documentation to local webspace" - cp -p dist/ftputil-${VERSION}.tar.gz ${WWW_DIR}/download + cp -p dist/ftputil-`cat VERSION`.tar.gz ${WWW_DIR}/download cp -p ftputil.html ${WWW_DIR}/python touch ${WWW_DIR}/python/python_software.tmpl @@ -125,5 +83,5 @@ python setup.py register -extdist: test dist debdist localcopy register +extdist: test dist localcopy register clean: Index: PKG-INFO =================================================================== --- PKG-INFO (revision 831:3335b387c7d2) +++ PKG-INFO (revision 601:e45298207132) @@ -1,11 +1,11 @@ -Metadata-Version: 1.0 +Metadata-Version: 1.1 Name: ftputil -Version: 2.4.2 +Version: 2.2b +Author: Stefan Schwarzer +Author-email: sschwarzer at sschwarzer net +Home-page: http://ftputil.sschwarzer.net/ +Download-url: http://ftputil.sschwarzer.net/trac/attachment/wiki/Download/ftputil-2.2b.tar.gz?format=raw Summary: High-level FTP client library (virtual filesystem and more) -Home-page: http://ftputil.sschwarzer.net/ -Author: Stefan Schwarzer -Author-email: sschwarzer@sschwarzer.net License: Open source (revised BSD license) -Download-URL: http://ftputil.sschwarzer.net/trac/attachment/wiki/Download/ftputil-2.4.2.tar.gz?format=raw Description: ftputil is a high-level FTP client library for the Python programming language. ftputil implements a virtual file system for accessing FTP servers, @@ -14,7 +14,7 @@ shutil modules. ftputil has convenience functions for conditional uploads and downloads, and handles FTP clients and servers in different timezones. -Keywords: FTP,client,virtual file system -Platform: Pure Python (Python version >= 2.3) -Classifier: Development Status :: 5 - Production/Stable +Keywords: FTP, client, virtual file system +Platform: Pure Python (Python version >= 2.1) +Classifier: Development Status :: 6 - Mature Classifier: Environment :: Other Environment Classifier: Intended Audience :: Developers @@ -23,4 +23,6 @@ Classifier: Programming Language :: Python Classifier: Topic :: Internet :: File Transfer Protocol (FTP) +Classifier: Topic :: Software Development Classifier: Topic :: Software Development :: Libraries :: Python Modules Classifier: Topic :: System :: Filesystems + Index: README.release =================================================================== --- README.release (revision 804:aaeff6db2578) +++ README.release (revision 519:b69badc3b25c) @@ -1,5 +1,3 @@ Things to do for a new release: - -- do commits - update info on new version in `README.txt` @@ -7,29 +5,13 @@ - write announcement in announcements.txt -- increase version number in `VERSION` (`ftputil_version.py` +- increase version number in `VERSION` (`ftputil_version.py` and `ftputil.txt` are handled by the `sed` invocation through - `make patch`) + `make dist`) -- update Debian-related packaging files in debian/custom, - including commits +- `make test && make dist` -- `make patch` - -- do outstanding commits due to patching - -- `make dist` (now includes tests) - -- tag release (tags are formed like "release2_0_3b2") - -- add new version to Trac issue tracker +- tag release - add new version to Download page on the website - -- mark corresponding milestone as completed - -- post announcement to ftputil mailing list - - -Only for non-alpha/beta releases: - update documentation on the website @@ -37,4 +19,8 @@ - register new version with PyPI -- send announcement to comp.lang.python.announce +- post announcement to ftputil mailing list +- for changes in minor version (i. e. not only a bugfix release), + send announcement to comp.lang.python; update file `announcements` + accordingly + Index: README.txt =================================================================== --- README.txt (revision 828:4a42bfcbe111) +++ README.txt (revision 597:0b945aa60ed3) @@ -16,44 +16,25 @@ ----------- -Since version 2.4.1 the following changed: +From version 2.1 to 2.2, the following has changed: -- Some FTP servers seem to have problems using *any* directory - argument which contains slashes. The new default for FTP commands - now is to change into the directory before actually invoking the - command on a relative path (report and fix suggestion by Nicola - Murino). +- Results of stat calls (also indirect calls in the submodule `path`, + i. e. isdir/isfile/islink, exists, getmtime etc.) are now cached + and reused. This results in remarkable speedups for many use cases. -- Calling the method ``FTPHost.stat_cache.resize`` with an argument 0 - caused an exception. This has been fixed; a zero cache size now - of course doesn't cache anything but doesn't lead to a traceback - either. +- The current directory is also locally cached, resulting in further + but usually lesser speedups. -- The installation script ``setup.py`` didn't work with the ``--home`` - option because it still tried to install the documentation in a - system directory (report by Albrecht Mühlenschulte). +- File-like objects generated via ``FTPHost.file`` now support the + iterator protocol (for line in some_file: ...). - As a side effect, when using the *global* installation, the - documentation is no longer installed in the ftputil package - directory but in a subdirectory ``doc`` of a directory determined by - Distutils. For example, on my system (Ubuntu 9.04) the documentation - files are put into ``/usr/local/doc``. +- The documentation has been updated accordingly. -Incompatibility notice ----------------------- +Possible incompatibilities: -When doing a system-wide installation, the documentation files are now -placed in another location than before (see last paragraph of the -previous section). This doesn't change anything regarding the use of -the library, though. +- This release requires at least Python 2.3. (Previous releases + worked with Python versions from 2.1 up.) -Both the ``xreadlines`` method and the long-deprecated direct access -of exceptions via the ``ftputil`` module (as in -``ftputil.PermanentError``) will be removed in ftputil *2.5*. Starting -with that version, exception classes will only be accessible via the -``ftp_error`` module. - -The distribution contains a small tool ``find_deprecated_code.py`` to -scan a directory for the deprecated uses. Invoke the program with the -``--help`` option to see a description. +- The method ``FTPHost.set_directory_format`` has been removed, + since the directory format is set automatically. Documentation @@ -61,5 +42,5 @@ The documentation for ftputil can be found in the file ftputil.txt -(reStructuredText format) or ftputil.html (recommended, generated +(reStructured Text format) or ftputil.html (recommended, generated from ftputil.txt). @@ -101,15 +82,4 @@ http://docs.python.org/inst/inst.html . -If you have easy_install installed, you can install the current -version of ftputil directly from the Python Package Index (PyPI) -without downloading the package explicitly. - -- Just type - - easy_install ftputil - - on the command line. You'll probably need root/administrator - privileges to do that (see above). - License ------- @@ -119,11 +89,9 @@ http://www.opensource.org/licenses/bsd-license.html ). -Authors -------- +Author +------ Stefan Schwarzer -Evan Prodromou (lrucache module) +Please provide feedback! It's surely appreciated. :-) -Please provide feedback! It's certainly appreciated. :-) - Index: TODO =================================================================== --- TODO (revision 774:8ca6b70c01d7) +++ TODO (revision 584:06608a909664) @@ -1,5 +1,10 @@ Planned: -- get ftputil included into Debian repositories +- write a ftputil_mirror.py script + a) as a usage example of ftputil + b) for benchmarking + c) as a useful tool +- support more directory formats (see mails from Oleg Broytmann) +- support time resolution in stat results (Oleg Broytmann) - add `ftp_session` module containing session classes (e. g. for passive mode connections) @@ -7,6 +12,8 @@ Ideas for future development: -- add an interface module for ftpparse (see conversation with - Oleg Broytmann) -> perhaps license issues -- WebDAV support? +- handle connection timeouts (Antonio Beamud Montero) +- merge with vpath project (Holger Krekel) +- merge in WebDAV support +- what about thread safety? (also have a look at `ftplib`) +- map FTP error numbers to os error numbers (ENOENT etc.)? Index: VERSION =================================================================== --- VERSION (revision 830:e90102d055af) +++ VERSION (revision 602:923443c9cd90) @@ -1,1 +1,1 @@ -2.4.2 +2.2b Index: __init__.py =================================================================== --- __init__.py (revision 677:16e76b94e500) +++ __init__.py (revision 539:a807c01d17f4) @@ -32,5 +32,5 @@ # $Id$ -# import everything into this namespace to comply with the old interface +# import all in this namespace to comply with the old interface # when ftputil was a single module from ftputil import * Index: _mock_ftplib.py =================================================================== --- _mock_ftplib.py (revision 807:9b759d5191e2) +++ _mock_ftplib.py (revision 604:22a0f6aa7364) @@ -1,3 +1,3 @@ -# Copyright (C) 2003-2009, Stefan Schwarzer +# Copyright (C) 2003-2004, Stefan Schwarzer # All rights reserved. # @@ -41,5 +41,4 @@ import ftplib -import posixpath import StringIO @@ -60,6 +59,6 @@ (not `_FTPFile` objects themselves!). - Contrary to `StringIO.StringIO` instances, `MockFile` objects can - be queried for their contents after they have been closed. + Unless `StringIO.StringIO` instances, `MockFile` objects can be + queried for their contents after they have been closed. """ def __init__(self, path, content=''): @@ -80,5 +79,5 @@ -class MockSocket(object): +class MockSocket: """ Mock class which is used to return something from @@ -98,5 +97,5 @@ -class MockSession(object): +class MockSession: """ Mock class which works like `ftplib.FTP` for the purpose of the @@ -134,5 +133,6 @@ drwxr-sr-x 6 45854 200 512 Sep 20 1999 scios2""", - '/home/dir with spaces': """\ + # = /home/dir with spaces + 'dir with spaces': """\ total 1 -rw-r--r-- 1 45854 200 4604 Jan 19 23:11 file with spaces""", @@ -167,4 +167,9 @@ self._transfercmds = 0 + def _remove_trailing_slash(self, path): + if path != '/' and path.endswith('/'): + path = path[:-1] + return path + def voidcmd(self, cmd): if DEBUG: @@ -174,6 +179,4 @@ elif cmd.startswith('TYPE '): return - elif cmd.startswith('SITE CHMOD'): - raise ftplib.error_perm("502 command not implemented") else: raise ftplib.error_perm @@ -182,22 +185,13 @@ return self.current_dir - def _remove_trailing_slash(self, path): - if path != '/' and path.endswith('/'): - path = path[:-1] - return path - - def _transform_path(self, path): - return posixpath.normpath(posixpath.join(self.pwd(), path)) - def cwd(self, path): - self.current_dir = self._transform_path(path) + path = self._remove_trailing_slash(path) + self.current_dir = path def dir(self, path, callback=None): - """Provide a callback function for processing each line of - a directory listing. Return nothing. - """ + "Provide a callback function with each line of a directory listing." if DEBUG: print 'dir: %s' % path - path = self._transform_path(path) + path = self._remove_trailing_slash(path) if not self.dir_contents.has_key(path): raise ftplib.error_perm Index: _test_base.py =================================================================== --- _test_base.py (revision 695:73ad91d942b3) +++ _test_base.py (revision 539:a807c01d17f4) @@ -1,3 +1,3 @@ -# Copyright (C) 2003-2004, Stefan Schwarzer +# Copyright (C) 2003, Stefan Schwarzer # All rights reserved. # Index: test_ftp_error.py =================================================================== --- _test_ftp_error.py (revision 772:2166f443d615) +++ (revision ) @@ -1,74 +1,0 @@ -# coding: utf-8 -# Copyright (C) 2002-2009, Stefan Schwarzer -# All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are -# met: -# -# - Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# -# - Redistributions in binary form must reproduce the above copyright -# notice, this list of conditions and the following disclaimer in the -# documentation and/or other materials provided with the distribution. -# -# - Neither the name of the above author nor the names of the -# contributors to the software may be used to endorse or promote -# products derived from this software without specific prior written -# permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -# ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR -# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -# LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -# NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -# $Id: $ - -import ftplib -import unittest - -import ftp_error - - -class TestFTPErrorArguments(unittest.TestCase): - def test_bytestring_argument(self): - # a umlaut as latin-1 character - os_error = ftp_error.FTPOSError("\xe4") - - def test_unicode_argument(self): - # a umlaut as unicode character - io_error = ftp_error.FTPIOError(u"\xe4") - - -class TestTryWithFTPError(unittest.TestCase): - def callee(self): - raise ftplib.error_perm() - - def test_try_with_oserror(self): - "Ensure the `ftplib` exception isn't used as `FTPOSError` argument." - try: - ftp_error._try_with_oserror(self.callee) - except ftp_error.FTPOSError, exc: - pass - self.failIf(exc.args and isinstance(exc.args[0], ftplib.error_perm)) - - def test_try_with_ioerror(self): - "Ensure the `ftplib` exception isn't used as `FTPIOError` argument." - try: - ftp_error._try_with_ioerror(self.callee) - except ftp_error.FTPIOError, exc: - pass - self.failIf(exc.args and isinstance(exc.args[0], ftplib.error_perm)) - - -if __name__ == '__main__': - unittest.main() - Index: test_ftp_file.py =================================================================== --- _test_ftp_file.py (revision 708:e0b3a3cde421) +++ (revision ) @@ -1,269 +1,0 @@ -# Copyright (C) 2002-2008, Stefan Schwarzer -# All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are -# met: -# -# - Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# -# - Redistributions in binary form must reproduce the above copyright -# notice, this list of conditions and the following disclaimer in the -# documentation and/or other materials provided with the distribution. -# -# - Neither the name of the above author nor the names of the -# contributors to the software may be used to endorse or promote -# products derived from this software without specific prior written -# permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -# ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR -# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -# LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -# NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -# $Id: $ - -import ftplib -import operator -import unittest - -import _mock_ftplib -import _test_base -import ftp_error -import ftp_file - - -# -# several customized `MockSession` classes -# -class ReadMockSession(_mock_ftplib.MockSession): - mock_file_content = 'line 1\r\nanother line\r\nyet another line' - -class AsciiReadMockSession(_mock_ftplib.MockSession): - mock_file_content = '\r\n'.join(map(str, range(20))) - -class InaccessibleDirSession(_mock_ftplib.MockSession): - _login_dir = '/inaccessible' - - def pwd(self): - return self._login_dir - - def cwd(self, dir): - if dir in (self._login_dir, self._login_dir + '/'): - raise ftplib.error_perm - else: - _mock_ftplib.MockSession.cwd(self, dir) - - -class TestFileOperations(unittest.TestCase): - """Test operations with file-like objects.""" - def test_inaccessible_dir(self): - """Test whether opening a file at an invalid location fails.""" - host = _test_base.ftp_host_factory( - session_factory=InaccessibleDirSession) - self.assertRaises(ftp_error.FTPIOError, host.file, - '/inaccessible/new_file', 'w') - - def test_caching(self): - """Test whether `_FTPFile` cache of `FTPHost` object works.""" - host = _test_base.ftp_host_factory() - self.assertEqual(len(host._children), 0) - path1 = 'path1' - path2 = 'path2' - # open one file and inspect cache - file1 = host.file(path1, 'w') - child1 = host._children[0] - self.assertEqual(len(host._children), 1) - self.failIf(child1._file.closed) - # open another file - file2 = host.file(path2, 'w') - child2 = host._children[1] - self.assertEqual(len(host._children), 2) - self.failIf(child2._file.closed) - # close first file - file1.close() - self.assertEqual(len(host._children), 2) - self.failUnless(child1._file.closed) - self.failIf(child2._file.closed) - # re-open first child's file - file1 = host.file(path1, 'w') - child1_1 = file1._host - # check if it's reused - self.failUnless(child1 is child1_1) - self.failIf(child1._file.closed) - self.failIf(child2._file.closed) - # close second file - file2.close() - self.failUnless(child2._file.closed) - - def test_write_to_directory(self): - """Test whether attempting to write to a directory fails.""" - host = _test_base.ftp_host_factory() - self.assertRaises(ftp_error.FTPIOError, host.file, - '/home/sschwarzer', 'w') - - def test_binary_write(self): - """Write binary data with `write`.""" - host = _test_base.ftp_host_factory() - data = '\000a\001b\r\n\002c\003\n\004\r\005' - output = host.file('dummy', 'wb') - output.write(data) - output.close() - child_data = _mock_ftplib.content_of('dummy') - expected_data = data - self.assertEqual(child_data, expected_data) - - def test_ascii_write(self): - """Write ASCII text with `write`.""" - host = _test_base.ftp_host_factory() - data = ' \nline 2\nline 3' - output = host.file('dummy', 'w') - output.write(data) - output.close() - child_data = _mock_ftplib.content_of('dummy') - expected_data = ' \r\nline 2\r\nline 3' - self.assertEqual(child_data, expected_data) - - def test_ascii_writelines(self): - """Write ASCII text with `writelines`.""" - host = _test_base.ftp_host_factory() - data = [' \n', 'line 2\n', 'line 3'] - backup_data = data[:] - output = host.file('dummy', 'w') - output.writelines(data) - output.close() - child_data = _mock_ftplib.content_of('dummy') - expected_data = ' \r\nline 2\r\nline 3' - self.assertEqual(child_data, expected_data) - # ensure that the original data was not modified - self.assertEqual(data, backup_data) - - def test_ascii_read(self): - """Read ASCII text with plain `read`.""" - host = _test_base.ftp_host_factory(session_factory=ReadMockSession) - input_ = host.file('dummy', 'r') - data = input_.read(0) - self.assertEqual(data, '') - data = input_.read(3) - self.assertEqual(data, 'lin') - data = input_.read(7) - self.assertEqual(data, 'e 1\nano') - data = input_.read() - self.assertEqual(data, 'ther line\nyet another line') - data = input_.read() - self.assertEqual(data, '') - input_.close() - # try it again with a more "problematic" string which - # makes several reads in the `read` method necessary - host = _test_base.ftp_host_factory(session_factory=AsciiReadMockSession) - expected_data = AsciiReadMockSession.mock_file_content.\ - replace('\r\n', '\n') - input_ = host.file('dummy', 'r') - data = input_.read(len(expected_data)) - self.assertEqual(data, expected_data) - - def test_binary_readline(self): - """Read binary data with `readline`.""" - host = _test_base.ftp_host_factory(session_factory=ReadMockSession) - input_ = host.file('dummy', 'rb') - data = input_.readline(3) - self.assertEqual(data, 'lin') - data = input_.readline(10) - self.assertEqual(data, 'e 1\r\n') - data = input_.readline(13) - self.assertEqual(data, 'another line\r') - data = input_.readline() - self.assertEqual(data, '\n') - data = input_.readline() - self.assertEqual(data, 'yet another line') - data = input_.readline() - self.assertEqual(data, '') - input_.close() - - def test_ascii_readline(self): - """Read ASCII text with `readline`.""" - host = _test_base.ftp_host_factory(session_factory=ReadMockSession) - input_ = host.file('dummy', 'r') - data = input_.readline(3) - self.assertEqual(data, 'lin') - data = input_.readline(10) - self.assertEqual(data, 'e 1\n') - data = input_.readline(13) - self.assertEqual(data, 'another line\n') - data = input_.readline() - self.assertEqual(data, 'yet another line') - data = input_.readline() - self.assertEqual(data, '') - input_.close() - - def test_ascii_readlines(self): - """Read ASCII text with `readlines`.""" - host = _test_base.ftp_host_factory(session_factory=ReadMockSession) - input_ = host.file('dummy', 'r') - data = input_.read(3) - self.assertEqual(data, 'lin') - data = input_.readlines() - self.assertEqual(data, ['e 1\n', 'another line\n', - 'yet another line']) - input_.close() - - def test_ascii_xreadlines(self): - """Read ASCII text with `xreadlines`.""" - host = _test_base.ftp_host_factory(session_factory=ReadMockSession) - # open file, skip some bytes - input_ = host.file('dummy', 'r') - data = input_.read(3) - xrl_obj = input_.xreadlines() - self.failUnless(xrl_obj.__class__ is ftp_file._XReadlines) - self.failUnless(xrl_obj._ftp_file.__class__ is ftp_file._FTPFile) - data = xrl_obj[0] - self.assertEqual(data, 'e 1\n') - # try to skip an index - self.assertRaises(RuntimeError, operator.__getitem__, xrl_obj, 2) - # continue reading - data = xrl_obj[1] - self.assertEqual(data, 'another line\n') - data = xrl_obj[2] - self.assertEqual(data, 'yet another line') - # try to read beyond EOF - self.assertRaises(IndexError, operator.__getitem__, xrl_obj, 3) - - def test_binary_iterator(self): - """Test the iterator interface of `FTPFile` objects.""" - host = _test_base.ftp_host_factory(session_factory=ReadMockSession) - input_ = host.file('dummy') - input_iterator = iter(input_) - self.assertEqual(input_iterator.next(), "line 1\n") - self.assertEqual(input_iterator.next(), "another line\n") - self.assertEqual(input_iterator.next(), "yet another line") - self.assertRaises(StopIteration, input_iterator.next) - input_.close() - - def test_ascii_iterator(self): - """Test the iterator interface of `FTPFile` objects.""" - host = _test_base.ftp_host_factory(session_factory=ReadMockSession) - input_ = host.file('dummy', 'rb') - input_iterator = iter(input_) - self.assertEqual(input_iterator.next(), "line 1\r\n") - self.assertEqual(input_iterator.next(), "another line\r\n") - self.assertEqual(input_iterator.next(), "yet another line") - self.assertRaises(StopIteration, input_iterator.next) - input_.close() - - def test_read_unknown_file(self): - """Test whether reading a file which isn't there fails.""" - host = _test_base.ftp_host_factory() - self.assertRaises(ftp_error.FTPIOError, host.file, 'notthere', 'r') - - -if __name__ == '__main__': - unittest.main() - Index: _test_ftp_path.py =================================================================== --- _test_ftp_path.py (revision 695:73ad91d942b3) +++ _test_ftp_path.py (revision 539:a807c01d17f4) @@ -1,3 +1,3 @@ -# Copyright (C) 2003-2007, Stefan Schwarzer +# Copyright (C) 2003-2004, Stefan Schwarzer # All rights reserved. # @@ -52,5 +52,5 @@ raise ftplib.error_perm("can't change into this directory") - + class TestPath(unittest.TestCase): """Test operations in `FTPHost.path`.""" Index: _test_ftp_stat.py =================================================================== --- _test_ftp_stat.py (revision 798:ff1b73253239) +++ _test_ftp_stat.py (revision 605:26bd1014b2b7) @@ -1,3 +1,3 @@ -# Copyright (C) 2003-2009, Stefan Schwarzer +# Copyright (C) 2003-2006, Stefan Schwarzer # All rights reserved. # @@ -48,5 +48,5 @@ stat = ftp_stat._Stat(host) # use Unix format parser explicitly - stat._parser = ftp_stat.UnixParser() + stat._parser = ftp_stat._UnixDirectoryParser() return stat @@ -61,5 +61,5 @@ -class TestParsers(unittest.TestCase): +class TestDirectoryParsers(unittest.TestCase): def _test_valid_lines(self, parser_class, lines, expected_stat_results): parser = parser_class() @@ -113,5 +113,5 @@ (2000, 5, 29, 0, 0, 0), None] ] - self._test_valid_lines(ftp_stat.UnixParser, lines, + self._test_valid_lines(ftp_stat._UnixDirectoryParser, lines, expected_stat_results) @@ -123,5 +123,5 @@ "xrwxr-sr-x 2 45854 200 51x May 4 2000 chemeng", ] - self._test_invalid_lines(ftp_stat.UnixParser, lines) + self._test_invalid_lines(ftp_stat._UnixDirectoryParser, lines) def test_alternative_unix_format(self): @@ -145,5 +145,5 @@ (2000, 5, 29, 0, 0, 0), None] ] - self._test_valid_lines(ftp_stat.UnixParser, lines, + self._test_valid_lines(ftp_stat._UnixDirectoryParser, lines, expected_stat_results) @@ -152,7 +152,5 @@ "07-27-01 11:16AM Test", "10-23-95 03:25PM WindowsXP", - "07-17-00 02:08PM 12266720 test.exe", - "07-17-09 12:08AM 12266720 test.exe", - "07-17-09 12:08PM 12266720 test.exe" + "07-17-00 02:08PM 12266720 test.exe" ] expected_stat_results = [ @@ -162,11 +160,8 @@ (1995, 10, 23, 15, 25, 0), None], [33024, None, None, None, None, None, 12266720, None, - (2000, 7, 17, 14, 8, 0), None], - [33024, None, None, None, None, None, 12266720, None, - (2009, 7, 17, 0, 8, 0), None], - [33024, None, None, None, None, None, 12266720, None, - (2009, 7, 17, 12, 8, 0), None] - ] - self._test_valid_lines(ftp_stat.MSParser, lines, expected_stat_results) + (2000, 7, 17, 14, 8, 0), None] + ] + self._test_valid_lines(ftp_stat._MSDirectoryParser, lines, + expected_stat_results) def test_invalid_ms_lines(self): @@ -176,5 +171,5 @@ "07-17-00 02:08AM 1226672x test.exe" ] - self._test_invalid_lines(ftp_stat.MSParser, lines) + self._test_invalid_lines(ftp_stat._MSDirectoryParser, lines) # @@ -226,5 +221,5 @@ host = _test_base.ftp_host_factory() # explicitly use Unix format parser - host._stat._parser = ftp_stat.UnixParser() + host._stat._parser = ftp_stat._UnixDirectoryParser() host.set_time_shift(supposed_time_shift) server_time = time.time() + supposed_time_shift + deviation @@ -273,6 +268,6 @@ """Test `lstat` for `/` . Note: `(l)stat` works by going one directory up and parsing - the output of an FTP `DIR` command. Unfortunately, it's not - possible to do this for the root directory `/`. + the output of an FTP `DIR` command. Unfortunately, it is not + possible to to this for the root directory `/`. """ self.assertRaises(ftp_error.RootDirError, self.stat.lstat, '/') @@ -344,7 +339,9 @@ """Test non-switching of parser format; stay with Unix.""" self.assertEqual(self.stat._allow_parser_switching, True) - self.failUnless(isinstance(self.stat._parser, ftp_stat.UnixParser)) + self.failUnless(isinstance(self.stat._parser, + ftp_stat._UnixDirectoryParser)) stat_result = self.stat.lstat("/home/sschwarzer/index.html") - self.failUnless(isinstance(self.stat._parser, ftp_stat.UnixParser)) + self.failUnless(isinstance(self.stat._parser, + ftp_stat._UnixDirectoryParser)) self.assertEqual(self.stat._allow_parser_switching, False) @@ -352,7 +349,9 @@ """Test switching of parser from Unix to MS format.""" self.assertEqual(self.stat._allow_parser_switching, True) - self.failUnless(isinstance(self.stat._parser, ftp_stat.UnixParser)) + self.failUnless(isinstance(self.stat._parser, + ftp_stat._UnixDirectoryParser)) stat_result = self.stat.lstat("/home/msformat/abcd.exe") - self.failUnless(isinstance(self.stat._parser, ftp_stat.MSParser)) + self.failUnless(isinstance(self.stat._parser, + ftp_stat._MSDirectoryParser)) self.assertEqual(self.stat._allow_parser_switching, False) self.assertEqual(stat_result._st_name, "abcd.exe") @@ -365,5 +364,4 @@ self.assertEqual(result, []) self.assertEqual(self.stat._allow_parser_switching, True) - self.failUnless(isinstance(self.stat._parser, ftp_stat.UnixParser)) Index: _test_ftp_stat_cache.py =================================================================== --- _test_ftp_stat_cache.py (revision 811:3064fdee60b0) +++ _test_ftp_stat_cache.py (revision 589:ea9c079e5f5f) @@ -36,5 +36,4 @@ import ftp_stat_cache -import _test_base @@ -117,11 +116,4 @@ self.cache.invalidate("/path2") - def test_cache_size_zero(self): - host = _test_base.ftp_host_factory() - host.stat_cache.resize(0) - # if bug #38 is present, this raises an `IndexError` - items = host.listdir(host.curdir) - self.assertEqual(items[:3], ['chemeng', 'download', 'image']) - if __name__ == '__main__': Index: test_ftp_sync.py =================================================================== --- _test_ftp_sync.py (revision 817:f1bd27bc249a) +++ (revision ) @@ -1,74 +1,0 @@ -# Copyright (C) 2007, Stefan Schwarzer -# All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are -# met: -# -# - Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# -# - Redistributions in binary form must reproduce the above copyright -# notice, this list of conditions and the following disclaimer in the -# documentation and/or other materials provided with the distribution. -# -# - Neither the name of the above author nor the names of the -# contributors to the software may be used to endorse or promote -# products derived from this software without specific prior written -# permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -# ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR -# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -# LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -# NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -# $Id$ - -import os -import shutil -import sys -import unittest - - -# assume the test subdirectories are or will be in the current directory -TEST_ROOT = os.getcwd() -sys.path.insert(0, os.path.dirname(TEST_ROOT)) - -import ftp_sync - - -class TestLocalToLocal(unittest.TestCase): - def setUp(self): - if not os.path.exists("test_empty"): - os.mkdir("test_empty") - if os.path.exists("test_target"): - shutil.rmtree("test_target") - os.mkdir("test_target") - - def test_sync_empty_dir(self): - source = ftp_sync.LocalHost() - target = ftp_sync.LocalHost() - syncer = ftp_sync.Syncer(source, target) - source_dir = os.path.join(TEST_ROOT, "test_empty") - target_dir = os.path.join(TEST_ROOT, "test_target") - syncer.sync(source_dir, target_dir) - - def test_source_with_and_target_without_slash(self): - source = ftp_sync.LocalHost() - target = ftp_sync.LocalHost() - syncer = ftp_sync.Syncer(source, target) - source_dir = os.path.join(TEST_ROOT, "test_source/") - target_dir = os.path.join(TEST_ROOT, "test_target") - syncer.sync(source_dir, target_dir) - - -if __name__ == '__main__': - unittest.main() - Index: _test_ftputil.py =================================================================== --- _test_ftputil.py (revision 807:9b759d5191e2) +++ _test_ftputil.py (revision 583:79fe7ae18727) @@ -1,3 +1,3 @@ -# Copyright (C) 2002-2009, Stefan Schwarzer +# Copyright (C) 2002-2006, Stefan Schwarzer # All rights reserved. # @@ -33,7 +33,8 @@ import ftplib +import operator import os -import posixpath import random +import stat import time import unittest @@ -42,7 +43,6 @@ import _test_base import ftp_error -import ftp_stat +import ftp_file import ftputil - # @@ -80,34 +80,9 @@ raise ftplib.error_perm -class RecursiveListingForDotAsPathSession(_mock_ftplib.MockSession): - dir_contents = { - ".": """\ -lrwxrwxrwx 1 staff 7 Aug 13 2003 bin -> usr/bin - -dev: -total 10 - -etc: -total 10 - -pub: -total 4 --rw-r--r-- 1 staff 74 Sep 25 2000 .message ----------- 1 staff 0 Aug 16 2003 .notar -drwxr-xr-x 12 ftp 512 Nov 23 2008 freeware - -usr: -total 4""", - - "": """\ -total 10 -lrwxrwxrwx 1 staff 7 Aug 13 2003 bin -> usr/bin -d--x--x--x 2 staff 512 Sep 24 2000 dev -d--x--x--x 3 staff 512 Sep 25 2000 etc -dr-xr-xr-x 3 staff 512 Oct 3 2000 pub -d--x--x--x 5 staff 512 Oct 3 2000 usr"""} - - def _transform_path(self, path): - return path +class ReadMockSession(_mock_ftplib.MockSession): + mock_file_content = 'line 1\r\nanother line\r\nyet another line' + +class AsciiReadMockSession(_mock_ftplib.MockSession): + mock_file_content = '\r\n'.join(map(str, range(20))) class BinaryDownloadMockSession(_mock_ftplib.MockSession): @@ -118,4 +93,16 @@ pass +class InaccessibleDirSession(_mock_ftplib.MockSession): + _login_dir = '/inaccessible' + + def pwd(self): + return self._login_dir + + def cwd(self, dir): + if dir in (self._login_dir, self._login_dir + '/'): + raise ftplib.error_perm + else: + _mock_ftplib.MockSession.cwd(self, dir) + # # customized `FTPHost` class for conditional upload/download tests @@ -131,14 +118,8 @@ class TimeShiftFTPHost(ftputil.FTPHost): class _Path: - def split(self, path): - return posixpath.split(path) def set_mtime(self, mtime): self._mtime = mtime def getmtime(self, file_name): return self._mtime - def join(self, *args): - return posixpath.join(*args) - def normpath(self, path): - return posixpath.normpath(path) def abspath(self, path): return "/home/sschwarzer/_ftputil_sync_" @@ -160,5 +141,5 @@ host = _test_base.ftp_host_factory() host.close() - self.assertEqual(host.closed, True) + self.assertEqual(host.closed, 1) self.assertEqual(host._children, []) @@ -171,68 +152,207 @@ -class TestSetParser(unittest.TestCase): - def test_set_parser(self): - """Test if the selected parser is used.""" - # this test isn't very practical but should help at least a bit ... - host = _test_base.ftp_host_factory() - # implicitly fix at Unix format - files = host.listdir("/home/sschwarzer") - self.assertEqual(files, ['chemeng', 'download', 'image', 'index.html', - 'os2', 'osup', 'publications', 'python', 'scios2']) - host.set_parser(ftp_stat.MSParser()) - files = host.listdir("/home/msformat/XPLaunch") - self.assertEqual(files, ['WindowsXP', 'XPLaunch', 'empty', - 'abcd.exe', 'O2KKeys.exe']) - self.assertEqual(host._stat._allow_parser_switching, False) - - -class TestCommandNotImplementedError(unittest.TestCase): - def test_command_not_implemented_error(self): - """ - Test if we get the anticipated exception if a command isn't - implemented by the server. - """ - host = _test_base.ftp_host_factory() - self.assertRaises(ftp_error.PermanentError, - host.chmod, "nonexistent", 0644) - # `CommandNotImplementedError` is a subclass of `PermanentError` - self.assertRaises(ftp_error.CommandNotImplementedError, - host.chmod, "nonexistent", 0644) - - -class TestRecursiveListingForDotAsPath(unittest.TestCase): - """Return a recursive directory listing when the path to list - is a dot. This is used to test for issue #33, see - http://ftputil.sschwarzer.net/trac/ticket/33 . - """ - def test_recursive_listing(self): - host = _test_base.ftp_host_factory( - session_factory=RecursiveListingForDotAsPathSession) - lines = host._dir(host.curdir) - self.assertEqual(lines[0], "total 10") - self.failUnless(lines[1].startswith("lrwxrwxrwx 1 staff")) - self.failUnless(lines[2].startswith("d--x--x--x 2 staff")) - host.close() - - def test_plain_listing(self): - host = _test_base.ftp_host_factory( - session_factory=RecursiveListingForDotAsPathSession) - lines = host._dir("") - self.assertEqual(lines[0], "total 10") - self.failUnless(lines[1].startswith("lrwxrwxrwx 1 staff")) - self.failUnless(lines[2].startswith("d--x--x--x 2 staff")) - host.close() - - def test_empty_string_instead_of_dot_workaround(self): - host = _test_base.ftp_host_factory( - session_factory=RecursiveListingForDotAsPathSession) - files = host.listdir(host.curdir) - self.assertEqual(files, ['bin', 'dev', 'etc', 'pub', 'usr']) - host.close() +class TestFileOperations(unittest.TestCase): + """Test operations with file-like objects.""" + def test_inaccessible_dir(self): + """Test whether opening a file at an invalid location fails.""" + host = _test_base.ftp_host_factory( + session_factory=InaccessibleDirSession) + self.assertRaises(ftp_error.FTPIOError, host.file, + '/inaccessible/new_file', 'w') + + def test_caching(self): + """Test whether `_FTPFile` cache of `FTPHost` object works.""" + host = _test_base.ftp_host_factory() + self.assertEqual(len(host._children), 0) + path1 = 'path1' + path2 = 'path2' + # open one file and inspect cache + file1 = host.file(path1, 'w') + child1 = host._children[0] + self.assertEqual(len(host._children), 1) + self.failIf(child1._file.closed) + # open another file + file2 = host.file(path2, 'w') + child2 = host._children[1] + self.assertEqual(len(host._children), 2) + self.failIf(child2._file.closed) + # close first file + file1.close() + self.assertEqual(len(host._children), 2) + self.failUnless(child1._file.closed) + self.failIf(child2._file.closed) + # re-open first child's file + file1 = host.file(path1, 'w') + child1_1 = file1._host + # check if it's reused + self.failUnless(child1 is child1_1) + self.failIf(child1._file.closed) + self.failIf(child2._file.closed) + # close second file + file2.close() + self.failUnless(child2._file.closed) + + def test_write_to_directory(self): + """Test whether attempting to write to a directory fails.""" + host = _test_base.ftp_host_factory() + self.assertRaises(ftp_error.FTPIOError, host.file, + '/home/sschwarzer', 'w') + + def test_binary_write(self): + """Write binary data with `write`.""" + host = _test_base.ftp_host_factory() + data = '\000a\001b\r\n\002c\003\n\004\r\005' + output = host.file('dummy', 'wb') + output.write(data) + output.close() + child_data = _mock_ftplib.content_of('dummy') + expected_data = data + self.assertEqual(child_data, expected_data) + + def test_ascii_write(self): + """Write ASCII text with `write`.""" + host = _test_base.ftp_host_factory() + data = ' \nline 2\nline 3' + output = host.file('dummy', 'w') + output.write(data) + output.close() + child_data = _mock_ftplib.content_of('dummy') + expected_data = ' \r\nline 2\r\nline 3' + self.assertEqual(child_data, expected_data) + + def test_ascii_writelines(self): + """Write ASCII text with `writelines`.""" + host = _test_base.ftp_host_factory() + data = [' \n', 'line 2\n', 'line 3'] + backup_data = data[:] + output = host.file('dummy', 'w') + output.writelines(data) + output.close() + child_data = _mock_ftplib.content_of('dummy') + expected_data = ' \r\nline 2\r\nline 3' + self.assertEqual(child_data, expected_data) + # ensure that the original data was not modified + self.assertEqual(data, backup_data) + + def test_ascii_read(self): + """Read ASCII text with plain `read`.""" + host = _test_base.ftp_host_factory(session_factory=ReadMockSession) + input_ = host.file('dummy', 'r') + data = input_.read(0) + self.assertEqual(data, '') + data = input_.read(3) + self.assertEqual(data, 'lin') + data = input_.read(7) + self.assertEqual(data, 'e 1\nano') + data = input_.read() + self.assertEqual(data, 'ther line\nyet another line') + data = input_.read() + self.assertEqual(data, '') + input_.close() + # try it again with a more "problematic" string which + # makes several reads in the `read` method necessary + host = _test_base.ftp_host_factory(session_factory=AsciiReadMockSession) + expected_data = AsciiReadMockSession.mock_file_content.\ + replace('\r\n', '\n') + input_ = host.file('dummy', 'r') + data = input_.read(len(expected_data)) + self.assertEqual(data, expected_data) + + def test_binary_readline(self): + """Read binary data with `readline`.""" + host = _test_base.ftp_host_factory(session_factory=ReadMockSession) + input_ = host.file('dummy', 'rb') + data = input_.readline(3) + self.assertEqual(data, 'lin') + data = input_.readline(10) + self.assertEqual(data, 'e 1\r\n') + data = input_.readline(13) + self.assertEqual(data, 'another line\r') + data = input_.readline() + self.assertEqual(data, '\n') + data = input_.readline() + self.assertEqual(data, 'yet another line') + data = input_.readline() + self.assertEqual(data, '') + input_.close() + + def test_ascii_readline(self): + """Read ASCII text with `readline`.""" + host = _test_base.ftp_host_factory(session_factory=ReadMockSession) + input_ = host.file('dummy', 'r') + data = input_.readline(3) + self.assertEqual(data, 'lin') + data = input_.readline(10) + self.assertEqual(data, 'e 1\n') + data = input_.readline(13) + self.assertEqual(data, 'another line\n') + data = input_.readline() + self.assertEqual(data, 'yet another line') + data = input_.readline() + self.assertEqual(data, '') + input_.close() + + def test_ascii_readlines(self): + """Read ASCII text with `readlines`.""" + host = _test_base.ftp_host_factory(session_factory=ReadMockSession) + input_ = host.file('dummy', 'r') + data = input_.read(3) + self.assertEqual(data, 'lin') + data = input_.readlines() + self.assertEqual(data, ['e 1\n', 'another line\n', + 'yet another line']) + input_.close() + + def test_ascii_xreadlines(self): + """Read ASCII text with `xreadlines`.""" + host = _test_base.ftp_host_factory(session_factory=ReadMockSession) + # open file, skip some bytes + input_ = host.file('dummy', 'r') + data = input_.read(3) + xrl_obj = input_.xreadlines() + self.failUnless(xrl_obj.__class__ is ftp_file._XReadlines) + self.failUnless(xrl_obj._ftp_file.__class__ is ftp_file._FTPFile) + data = xrl_obj[0] + self.assertEqual(data, 'e 1\n') + # try to skip an index + self.assertRaises(RuntimeError, operator.__getitem__, xrl_obj, 2) + # continue reading + data = xrl_obj[1] + self.assertEqual(data, 'another line\n') + data = xrl_obj[2] + self.assertEqual(data, 'yet another line') + # try to read beyond EOF + self.assertRaises(IndexError, operator.__getitem__, xrl_obj, 3) + + def test_binary_iterator(self): + """Test the iterator interface of `FTPFile` objects.""" + host = _test_base.ftp_host_factory(session_factory=ReadMockSession) + input_ = host.file('dummy') + input_iterator = iter(input_) + self.assertEqual(input_iterator.next(), "line 1\n") + self.assertEqual(input_iterator.next(), "another line\n") + self.assertEqual(input_iterator.next(), "yet another line") + self.assertRaises(StopIteration, input_iterator.next) + input_.close() + + def test_ascii_iterator(self): + """Test the iterator interface of `FTPFile` objects.""" + host = _test_base.ftp_host_factory(session_factory=ReadMockSession) + input_ = host.file('dummy', 'rb') + input_iterator = iter(input_) + self.assertEqual(input_iterator.next(), "line 1\r\n") + self.assertEqual(input_iterator.next(), "another line\r\n") + self.assertEqual(input_iterator.next(), "yet another line") + self.assertRaises(StopIteration, input_iterator.next) + input_.close() + + def test_read_unknown_file(self): + """Test whether reading a file which isn't there fails.""" + host = _test_base.ftp_host_factory() + self.assertRaises(ftp_error.FTPIOError, host.file, 'notthere', 'r') class TestUploadAndDownload(unittest.TestCase): """Test ASCII upload and binary download as examples.""" - def generate_ascii_file(self, data, filename): """Generate an ASCII data file.""" @@ -355,5 +475,5 @@ # use private bound method rounded_time_shift = host._FTPHost__rounded_time_shift - # pairs consisting of original value and expected result + # original value, expected result test_data = [ (0, 0), (0.1, 0), (-0.1, 0), (1500, 0), (-1500, 0), Index: test_public_servers.py =================================================================== --- _test_public_servers.py (revision 810:031617cedff3) +++ (revision ) @@ -1,195 +1,0 @@ -# Copyright (C) 2009, Stefan Schwarzer -# All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are -# met: -# -# - Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# -# - Redistributions in binary form must reproduce the above copyright -# notice, this list of conditions and the following disclaimer in the -# documentation and/or other materials provided with the distribution. -# -# - Neither the name of the above author nor the names of the -# contributors to the software may be used to endorse or promote -# products derived from this software without specific prior written -# permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -# ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR -# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -# LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -# NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -# $Id: $ - -import os -import subprocess -import unittest - -import ftputil - - -def email_address(): - """ - Return the email address used to identify the client to an - FTP server. - - If the hostname is "warpy", use my (Stefan's) email address, - else try to use the content of the $EMAIL environment variable. - If that doesn't exist, use a dummy address. - """ - try: - fobj = open("/etc/hostname") - hostname = fobj.read().strip() - finally: - fobj.close() - if hostname == "warpy": - email = "sschwarzer@sschwarzer.net" - else: - dummy_address = "anonymous@example.com" - email = os.environ.get("EMAIL", dummy_address) - if not email: - # environment variable exists but content is an empty string - email = dummy_address - return email - -EMAIL = email_address() - - -def ftp_client_listing(server, directory): - """ - Log into the FTP server `server` using the command line - client, then change to the `directory` and retrieve a - listing with "dir". Return the list of items found as the - an `os.listdir` would return it. - """ - # the -n option prevents an auto-login - ftp_popen = subprocess.Popen(["ftp", "-n", server], - stdin=subprocess.PIPE, - stdout=subprocess.PIPE, - universal_newlines=True) - commands = ["user anonymous %s" % EMAIL, "dir", "bye"] - if directory: - # change to this directory before calling "dir" - commands.insert(1, "cd %s" % directory) - input_ = "\n".join(commands) - stdout, stderr = ftp_popen.communicate(input_) - # collect the directory/file names from the listing's text - names = [] - for line in stdout.strip().split("\n"): - if line.startswith("total "): - continue - parts = line.split() - if parts[-2] == "->": - # most probably a link - name = parts[-3] - else: - name = parts[-1] - names.append(name) - # remove entries for current and parent directory - names = [name for name in names if name not in (".", "..")] - return names - - -class TestPublicServers(unittest.TestCase): - """ - Get directory listings from various public FTP servers - with a command line client and ftputil and compare both. - - An important aspect is to test different "spellings" of - the same directory. For example, to list the root directory - which is usually set after login, use "" (nothing), ".", - "/", "/.", "./.", "././", "..", "../.", "../.." etc. - - The command line client "ftp" has to be in the path. - """ - - # Implementation note: - # - # I (Stefan) implement the code so it works with Ubuntu's - # client. Other clients may work or not. If you have problems - # testing some other client, please send me a (small) patch. - # Keep in mind that I don't plan supporting as many FTP - # clients as servers. ;-) - - # list of pairs with server name and a directory "guaranteed - # to exist" under the login directory which is assumed to be - # the root directory - servers = [# Posix format - ("ftp.gnome.org", "pub"), - ("ftp.debian.org", "debian"), - ("ftp.sunfreeware.com", "pub"), - ("ftp.chello.nl", "pub"), - ("ftp.heanet.ie", "pub"), - # DOS/Microsoft format - ("ftp.microsoft.com", "deskapps")] - - # This data structure contains the initial directories "." and - # "DIR" (which will be replaced by a valid directory name for - # each server). The list after the initial directory contains - # paths that will be queried after changing into the initial - # directory. All items in these lists are actually supposed to - # yield the same directory contents. - paths_table = [ - (".", ["", ".", "/", "/.", "./.", "././", "..", "../.", "../..", - "DIR/..", "/DIR/../.", "/DIR/../.."]), - ("DIR", ["", ".", "/DIR", "/DIR/", "../DIR", "../../DIR"]) - ] - - def inner_test_server(self, server, initial_directory, paths): - """ - Test one server for one initial directory. - - Connect to the server `server`; if the string argument - `initial_directory` has a true value, change to this - directory. Then iterate over all strings in the sequence - `paths`, comparing the results of a listdir call with the - listing from the command line client. - """ - canonical_names = ftp_client_listing(server, initial_directory) - host = ftputil.FTPHost(server, "anonymous", EMAIL) - try: - host.chdir(initial_directory) - for path in paths: - path = path.replace("DIR", initial_directory) - # make sure that we don't recycle directory entries, i. e. - # really repeatedly retrieve the directory contents - # (shouldn't happen anyway with the current implementation) - host.stat_cache.clear() - names = host.listdir(path) - failure_message = "For server %s, directory %s: %s != %s" % \ - (server, initial_directory, names, - canonical_names) - self.assertEqual(names, canonical_names) - finally: - host.close() - - def test_servers(self): - """ - Test all servers in `self.servers`. - - For each server, get the listings for the login directory and - one other directory which is known to exist. Use different - "spellings" to retrieve each list via ftputil and compare with - the results gotten with the command line client. - """ - for server, actual_initial_directory in self.servers: - for initial_directory, paths in self.paths_table: - initial_directory = initial_directory.replace( - "DIR", actual_initial_directory) - print server, initial_directory - self.inner_test_server(server, initial_directory, paths) - - -if __name__ == '__main__': - unittest.main() - Index: test_python2_4.py =================================================================== --- _test_python2_4.py (revision 791:06f681345c6e) +++ (revision ) @@ -1,53 +1,0 @@ -# Copyright (C) 2003-2009, Stefan Schwarzer -# All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are -# met: -# -# - Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# -# - Redistributions in binary form must reproduce the above copyright -# notice, this list of conditions and the following disclaimer in the -# documentation and/or other materials provided with the distribution. -# -# - Neither the name of the above author nor the names of the -# contributors to the software may be used to endorse or promote -# products derived from this software without specific prior written -# permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -# ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR -# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -# LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -# NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -# $Id: _test_real_ftp.py 810 2009-04-05 18:09:43Z schwa $ - -import unittest - -import ftp_error - - -class Python24(unittest.TestCase): - """Test for faults which occur only with Python 2.4 (possibly below).""" - - def test_exception_base_class(self): - try: - raise ftp_error.FTPOSError("") - except TypeError: - self.fail("can't use super in classic class") - except ftp_error.FTPOSError: - # everything's fine - pass - - -if __name__ == '__main__': - unittest.main() Index: _test_real_ftp.py =================================================================== --- _test_real_ftp.py (revision 820:8cb525920db5) +++ _test_real_ftp.py (revision 591:3376329092e5) @@ -1,3 +1,3 @@ -# Copyright (C) 2003-2009, Stefan Schwarzer +# Copyright (C) 2003-2006, Stefan Schwarzer # All rights reserved. # @@ -35,9 +35,7 @@ import getpass -import operator import os import time import unittest -import stat import sys @@ -46,4 +44,7 @@ from ftputil import ftp_stat +# difference between local times of server and client; if 0.0, server +# and client are in the same timezone +EXPECTED_TIME_SHIFT = 0.0 def get_login_data(): @@ -58,99 +59,33 @@ return "localhost", 'ftptest', 'd605581757de5eb56d568a4419f4126e' -def utc_local_time_shift(): - """ - Return the expected time shift in seconds assuming the server - uses UTC in its listings and the client uses local time. - - This is needed because Pure-FTPd meanwhile seems to insist that - the displayed time for files is in UTC. - """ - utc_tuple = time.gmtime() - localtime_tuple = time.localtime() - # to calculate the correct times shift, we need to ignore the - # DST component in the localtime tuple, i. e. set it to 0 - localtime_tuple = localtime_tuple[:-1] + (0,) - time_shift_in_seconds = time.mktime(utc_tuple) - \ - time.mktime(localtime_tuple) - # to be safe, round the above value to units of 3600 s (1 h) - return round(time_shift_in_seconds / 3600.0) * 3600 - -# difference between local times of server and client; if 0.0, server -# and client use the same timezone -EXPECTED_TIME_SHIFT = utc_local_time_shift() - - -class Cleaner(object): - """This class helps to remove directories and files which - might be left behind if a test fails in unexpected ways. - """ - - def __init__(self, host): - # the test class (probably `RealFTPTest`) and the helper - # class share the same `FTPHost` object - self._host = host - self._ftp_items = [] - - def add_dir(self, path): - """Schedule a directory with path `path` for removal.""" - self._ftp_items.append(('d', self._host.path.abspath(path))) - - def add_file(self, path): - """Schedule a file with path `path` for removal.""" - self._ftp_items.append(('f', self._host.path.abspath(path))) - - def clean(self): - """Remove the directories and files previously remembered. - The removal works in reverse order of the scheduling with - `add_dir` and `add_file`. - - Errors due to a removal are ignored. - """ - self._host.chdir("/") - # code should work with Python 2.3 - self._ftp_items.reverse() - for type_, path in self._ftp_items: - try: - if type_ == 'd': - # if something goes wrong in `rmtree` we might - # leave a mess behind - self._host.rmtree(path) - elif type_ == 'f': - # minor mess if `remove` fails - self._host.remove(path) - except ftp_error.FTPError: - pass - class RealFTPTest(unittest.TestCase): def setUp(self): self.host = ftputil.FTPHost(server, user, password) - self.cleaner = Cleaner(self.host) def tearDown(self): - self.cleaner.clean() self.host.close() - # - # helper methods - # def make_file(self, path): - self.cleaner.add_file(path) file_ = self.host.file(path, 'wb') file_.close() - def make_local_file(self): - fobj = file('_localfile_', 'wb') - fobj.write("abc\x12\x34def\t") - fobj.close() - - # - # `mkdir`, `makedirs`, `rmdir` and `rmtree` - # + def test_open_for_reading(self): + # test for issue #17, http://ftputil.sschwarzer.net/trac/ticket/17 + file1 = self.host.file("debian-keyring.tar.gz", 'rb') + file1.close() + # make sure that there are no problems if the connection is reused + file2 = self.host.file("debian-keyring.tar.gz", 'rb') + file2.close() + self.failUnless(file1._session is file2._session) + + def test_time_shift(self): + self.host.synchronize_times() + self.assertEqual(self.host.time_shift(), EXPECTED_TIME_SHIFT) + def test_mkdir_rmdir(self): host = self.host dir_name = "_testdir_" file_name = host.path.join(dir_name, "_nonempty_") - self.cleaner.add_dir(dir_name) # make dir and check if it's there host.mkdir(dir_name) @@ -158,5 +93,4 @@ self.failIf(dir_name not in files) # try to remove non-empty directory - self.cleaner.add_file(file_name) non_empty = host.file(file_name, "w") non_empty.close() @@ -170,5 +104,5 @@ except ftp_error.PermanentError, exc: self.failUnless(str(exc).startswith( - "remove/unlink can only delete files")) + "remove/unlink can only delete files")) else: self.failIf(True, "we shouldn't have come here") @@ -181,51 +115,18 @@ def test_makedirs_without_existing_dirs(self): host = self.host - # no `_dir1_` yet - self.failIf('_dir1_' in host.listdir(host.curdir)) + # no `dir1` yet + self.failIf('dir1' in host.listdir(host.curdir)) # vanilla case, all should go well - host.makedirs('_dir1_/dir2/dir3/dir4') - self.cleaner.add_dir('_dir1_') + host.makedirs('dir1/dir2/dir3/dir4') # check host - self.failUnless(host.path.isdir('_dir1_')) - self.failUnless(host.path.isdir('_dir1_/dir2')) - self.failUnless(host.path.isdir('_dir1_/dir2/dir3')) - self.failUnless(host.path.isdir('_dir1_/dir2/dir3/dir4')) - - def test_makedirs_from_non_root_directory(self): - # this is a testcase for issue #22, see - # http://ftputil.sschwarzer.net/trac/ticket/22 - host = self.host - # no `_dir1_` and `_dir2_` yet - self.failIf('_dir1_' in host.listdir(host.curdir)) - self.failIf('_dir2_' in host.listdir(host.curdir)) - # part 1: try to make directories starting from `_dir1_` - # make and change to non-root directory - self.cleaner.add_dir("_dir1_") - host.mkdir('_dir1_') - host.chdir('_dir1_') - host.makedirs('_dir2_/_dir3_') - # test for expected directory hierarchy - self.failUnless(host.path.isdir('/_dir1_')) - self.failUnless(host.path.isdir('/_dir1_/_dir2_')) - self.failUnless(host.path.isdir('/_dir1_/_dir2_/_dir3_')) - self.failIf(host.path.isdir('/_dir1_/_dir1_')) - # remove all but the directory we're in - host.rmdir('/_dir1_/_dir2_/_dir3_') - host.rmdir('/_dir1_/_dir2_') - # part 2: try to make directories starting from root - self.cleaner.add_dir("/_dir2_") - host.makedirs('/_dir2_/_dir3_') - # test for expected directory hierarchy - self.failUnless(host.path.isdir('/_dir2_')) - self.failUnless(host.path.isdir('/_dir2_/_dir3_')) - self.failIf(host.path.isdir('/_dir1_/_dir2_')) - - def test_makedirs_from_non_root_directory_fake_windows_os(self): - saved_sep = os.sep - os.sep = '\\' - try: - self.test_makedirs_from_non_root_directory() - finally: - os.sep = saved_sep + self.failUnless(host.path.isdir('dir1')) + self.failUnless(host.path.isdir('dir1/dir2')) + self.failUnless(host.path.isdir('dir1/dir2/dir3')) + self.failUnless(host.path.isdir('dir1/dir2/dir3/dir4')) + # clean up + host.rmdir('dir1/dir2/dir3/dir4') + host.rmdir('dir1/dir2/dir3') + host.rmdir('dir1/dir2') + host.rmdir('dir1') def test_makedirs_of_existing_directory(self): @@ -236,21 +137,24 @@ def test_makedirs_with_file_in_the_way(self): host = self.host - self.cleaner.add_dir('_dir1_') - host.mkdir('_dir1_') - self.make_file('_dir1_/file1') + host.mkdir('dir1') + self.make_file('dir1/file1') # try it + self.assertRaises(ftp_error.PermanentError, host.makedirs, 'dir1/file1') self.assertRaises(ftp_error.PermanentError, host.makedirs, - '_dir1_/file1') - self.assertRaises(ftp_error.PermanentError, host.makedirs, - '_dir1_/file1/dir2') + 'dir1/file1/dir2') + # clean up + host.unlink('dir1/file1') + host.rmdir('dir1') def test_makedirs_with_existing_directory(self): host = self.host - self.cleaner.add_dir("_dir1_") - host.mkdir('_dir1_') - host.makedirs('_dir1_/dir2') + host.mkdir('dir1') + host.makedirs('dir1/dir2') # check - self.failUnless(host.path.isdir('_dir1_')) - self.failUnless(host.path.isdir('_dir1_/dir2')) + self.failUnless(host.path.isdir('dir1')) + self.failUnless(host.path.isdir('dir1/dir2')) + # clean up + host.rmdir('dir1/dir2') + host.rmdir('dir1') def test_makedirs_in_non_writable_directory(self): @@ -262,5 +166,4 @@ def test_makedirs_with_writable_directory_at_end(self): host = self.host - self.cleaner.add_dir('rootdir2/dir2') # preparation: `rootdir2` exists but is only writable by root; # `dir2` is writable by regular ftp user @@ -268,32 +171,32 @@ host.makedirs('rootdir2/dir2') host.makedirs('rootdir2/dir2/dir3') + # clean up + host.rmdir('rootdir2/dir2/dir3') def test_rmtree_without_error_handler(self): host = self.host # build a tree - self.cleaner.add_dir('_dir1_') - host.makedirs('_dir1_/dir2') - self.make_file('_dir1_/file1') - self.make_file('_dir1_/file2') - self.make_file('_dir1_/dir2/file3') - self.make_file('_dir1_/dir2/file4') + host.makedirs('dir1/dir2') + self.make_file('dir1/file1') + self.make_file('dir1/file2') + self.make_file('dir1/dir2/file3') + self.make_file('dir1/dir2/file4') # try to remove a _file_ with `rmtree` - self.assertRaises(ftp_error.PermanentError, host.rmtree, '_dir1_/file2') + self.assertRaises(ftp_error.PermanentError, host.rmtree, 'dir1/file2') # remove dir2 - host.rmtree('_dir1_/dir2') - self.failIf(host.path.exists('_dir1_/dir2')) - self.failUnless(host.path.exists('_dir1_/file2')) - # remake dir2 and remove _dir1_ - host.mkdir('_dir1_/dir2') - self.make_file('_dir1_/dir2/file3') - self.make_file('_dir1_/dir2/file4') - host.rmtree('_dir1_') - self.failIf(host.path.exists('_dir1_')) + host.rmtree('dir1/dir2') + self.failIf(host.path.exists('dir1/dir2')) + self.failUnless(host.path.exists('dir1/file2')) + # remake dir2 and remove dir1 + host.mkdir('dir1/dir2') + self.make_file('dir1/dir2/file3') + self.make_file('dir1/dir2/file4') + host.rmtree('dir1') + self.failIf(host.path.exists('dir1')) def test_rmtree_with_error_handler(self): host = self.host - self.cleaner.add_dir('_dir1_') - host.mkdir('_dir1_') - self.make_file('_dir1_/file1') + host.mkdir('dir1') + self.make_file('dir1/file1') # prepare error "handler" log = [] @@ -301,93 +204,20 @@ log.append(args) # try to remove a file as root "directory" - host.rmtree('_dir1_/file1', ignore_errors=True, onerror=error_handler) + host.rmtree('dir1/file1', ignore_errors=True, onerror=error_handler) self.assertEqual(log, []) - host.rmtree('_dir1_/file1', ignore_errors=False, onerror=error_handler) + host.rmtree('dir1/file1', ignore_errors=False, onerror=error_handler) self.assertEqual(log[0][0], host.listdir) - self.assertEqual(log[0][1], '_dir1_/file1') + self.assertEqual(log[0][1], 'dir1/file1') self.assertEqual(log[1][0], host.rmdir) - self.assertEqual(log[1][1], '_dir1_/file1') - host.rmtree('_dir1_') + self.assertEqual(log[1][1], 'dir1/file1') + host.rmtree('dir1') # try to remove a non-existent directory del log[:] - host.rmtree('_dir1_', ignore_errors=False, onerror=error_handler) + host.rmtree('dir1', ignore_errors=False, onerror=error_handler) self.assertEqual(log[0][0], host.listdir) - self.assertEqual(log[0][1], '_dir1_') + self.assertEqual(log[0][1], 'dir1') self.assertEqual(log[1][0], host.rmdir) - self.assertEqual(log[1][1], '_dir1_') - - # - # directory tree walking - # - def test_walk_topdown(self): - # preparation: build tree in directory `walk_test` - host = self.host - expected = [ - ('walk_test', ['dir1', 'dir2', 'dir3'], ['file4']), - ('walk_test/dir1', ['dir11', 'dir12'], []), - ('walk_test/dir1/dir11', [], []), - ('walk_test/dir1/dir12', ['dir123'], ['file121', 'file122']), - ('walk_test/dir1/dir12/dir123', [], ['file1234']), - ('walk_test/dir2', [], []), - ('walk_test/dir3', ['dir33'], ['file31', 'file32']), - ('walk_test/dir3/dir33', [], []), - ] - # collect data, using `walk` - actual = [] - for items in host.walk('walk_test'): - actual.append(items) - # compare with expected results - self.assertEqual(len(actual), len(expected)) - for index in range(len(actual)): - self.assertEqual(actual[index], expected[index]) - - def test_walk_depth_first(self): - # preparation: build tree in directory `walk_test` - host = self.host - expected = [ - ('walk_test/dir1/dir11', [], []), - ('walk_test/dir1/dir12/dir123', [], ['file1234']), - ('walk_test/dir1/dir12', ['dir123'], ['file121', 'file122']), - ('walk_test/dir1', ['dir11', 'dir12'], []), - ('walk_test/dir2', [], []), - ('walk_test/dir3/dir33', [], []), - ('walk_test/dir3', ['dir33'], ['file31', 'file32']), - ('walk_test', ['dir1', 'dir2', 'dir3'], ['file4']) - ] - # collect data, using `walk` - actual = [] - for items in host.walk('walk_test', topdown=False): - actual.append(items) - # compare with expected results - self.assertEqual(len(actual), len(expected)) - for index in range(len(actual)): - self.assertEqual(actual[index], expected[index]) - - # - # renaming - # - def test_rename(self): - host = self.host - # make sure the target of the renaming operation is removed - self.cleaner.add_file('_testfile2_') - self.make_file("_testfile1_") - host.rename('_testfile1_', '_testfile2_') - self.failIf(host.path.exists('_testfile1_')) - self.failUnless(host.path.exists('_testfile2_')) - host.remove('_testfile2_') - - def test_rename_with_spaces_in_directory(self): - host = self.host - dir_name = "_dir with spaces_" - self.cleaner.add_dir(dir_name) - host.mkdir(dir_name) - self.make_file(dir_name + "/testfile1") - host.rename(dir_name + "/testfile1", dir_name + "/testfile2") - self.failIf(host.path.exists(dir_name + "/testfile1")) - self.failUnless(host.path.exists(dir_name + "/testfile2")) - - # - # stat'ing - # + self.assertEqual(log[1][1], 'dir1') + def test_stat(self): host = self.host @@ -395,5 +225,4 @@ file_name = host.path.join(dir_name, "_nonempty_") # make a directory and a file in it - self.cleaner.add_dir(dir_name) host.mkdir(dir_name) fobj = host.file(file_name, "wb") @@ -417,16 +246,77 @@ calculated_time_shift = server_mtime - client_mtime self.failIf(abs(calculated_time_shift-host.time_shift()) > 120) - -# def test_special_broken_link(self): -# # test for ticket #39 -# # this test currently fails; I guess I'll postpone it until -# # at least ftputil 2.5 -# host = self.host -# broken_link_name = os.path.join("dir_with_broken_link", "nonexistent") -# self.assertEqual(host.lstat(broken_link_name)._st_target, -# "../nonexistent/nonexistent") -# self.assertEqual(bool(host.path.isdir(broken_link_name)), False) -# self.assertEqual(bool(host.path.isfile(broken_link_name)), False) -# self.assertEqual(bool(host.path.islink(broken_link_name)), True) + # clean up + host.unlink(file_name) + host.rmdir(dir_name) + + def make_local_file(self): + fobj = file("_localfile_", "wb") + fobj.write("abc\x12\x34def\t") + fobj.close() + + def test_upload(self): + host = self.host + # make local file and upload it + self.make_local_file() + # wait; else small time differences between client and server + # actually could trigger the update + time.sleep(60) + host.upload("_localfile_", "_remotefile_", "b") + # retry; shouldn't be uploaded + uploaded = host.upload_if_newer("_localfile_", "_remotefile_", "b") + self.assertEqual(uploaded, False) + # rewrite the local file + self.make_local_file() + time.sleep(60) + # retry; should be uploaded + uploaded = host.upload_if_newer("_localfile_", "_remotefile_", "b") + self.assertEqual(uploaded, True) + # clean up + os.unlink("_localfile_") + host.unlink("_remotefile_") + + def test_walk_topdown(self): + # preparation: build tree in directory `walk_test` + host = self.host + expected = [ + ('walk_test', ['dir1', 'dir2', 'dir3'], ['file4']), + ('walk_test/dir1', ['dir11', 'dir12'], []), + ('walk_test/dir1/dir11', [], []), + ('walk_test/dir1/dir12', ['dir123'], ['file121', 'file122']), + ('walk_test/dir1/dir12/dir123', [], ['file1234']), + ('walk_test/dir2', [], []), + ('walk_test/dir3', ['dir33'], ['file31', 'file32']), + ('walk_test/dir3/dir33', [], []), + ] + # collect data, using `walk` + actual = [] + for items in host.walk('walk_test'): + actual.append(items) + # compare with expected results + self.assertEqual(len(actual), len(expected)) + for index in range(len(actual)): + self.assertEqual(actual[index], expected[index]) + + def test_walk_depth_first(self): + # preparation: build tree in directory `walk_test` + host = self.host + expected = [ + ('walk_test/dir1/dir11', [], []), + ('walk_test/dir1/dir12/dir123', [], ['file1234']), + ('walk_test/dir1/dir12', ['dir123'], ['file121', 'file122']), + ('walk_test/dir1', ['dir11', 'dir12'], []), + ('walk_test/dir2', [], []), + ('walk_test/dir3/dir33', [], []), + ('walk_test/dir3', ['dir33'], ['file31', 'file32']), + ('walk_test', ['dir1', 'dir2', 'dir3'], ['file4']) + ] + # collect data, using `walk` + actual = [] + for items in host.walk('walk_test', topdown=False): + actual.append(items) + # compare with expected results + self.assertEqual(len(actual), len(expected)) + for index in range(len(actual)): + self.assertEqual(actual[index], expected[index]) def test_concurrent_access(self): @@ -447,142 +337,7 @@ host1.stat_cache.invalidate(absolute_path) self.assertRaises(ftp_error.PermanentError, host1.stat, "_testfile_") - - # - # `upload` (including time shift test) - # - def test_time_shift(self): - self.host.synchronize_times() - self.assertEqual(self.host.time_shift(), EXPECTED_TIME_SHIFT) - - def test_upload(self): - host = self.host - host.synchronize_times() - # make local file and upload it - self.make_local_file() - # wait; else small time differences between client and server - # actually could trigger the update - time.sleep(65) - try: - self.cleaner.add_file('_remotefile_') - host.upload('_localfile_', '_remotefile_', 'b') - # retry; shouldn't be uploaded - uploaded = host.upload_if_newer('_localfile_', '_remotefile_', 'b') - self.assertEqual(uploaded, False) - # rewrite the local file - self.make_local_file() - time.sleep(65) - # retry; should be uploaded now - uploaded = host.upload_if_newer('_localfile_', '_remotefile_', 'b') - self.assertEqual(uploaded, True) - finally: - # clean up - os.unlink('_localfile_') - - # - # remove/unlink - # - def test_remove_non_existent_item(self): - host = self.host - self.assertRaises(ftp_error.PermanentError, host.remove, "nonexistent") - - def test_remove_existent_file(self): - self.cleaner.add_file('_testfile_') - self.make_file('_testfile_') - host = self.host - self.failUnless(host.path.isfile('_testfile_')) - host.remove('_testfile_') - self.failIf(host.path.exists('_testfile_')) - - # - # `chmod` - # - def assert_mode(self, path, expected_mode): - """Return an integer containing the allowed bits in the - mode change command. - - The `FTPHost` object to test against is `self.host`. - """ - full_mode = self.host.stat(path).st_mode - # remove flags we can't set via `chmod` - # allowed flags according to Python documentation - # http://docs.python.org/lib/os-file-dir.html - allowed_flags = [stat.S_ISUID, stat.S_ISGID, stat.S_ENFMT, - stat.S_ISVTX, stat.S_IREAD, stat.S_IWRITE, stat.S_IEXEC, - stat.S_IRWXU, stat.S_IRUSR, stat.S_IWUSR, stat.S_IXUSR, - stat.S_IRWXG, stat.S_IRGRP, stat.S_IWGRP, stat.S_IXGRP, - stat.S_IRWXO, stat.S_IROTH, stat.S_IWOTH, stat.S_IXOTH] - allowed_mask = reduce(operator.or_, allowed_flags) - mode = full_mode & allowed_mask - self.assertEqual(mode, expected_mode, - "mode %s != %s" % (oct(mode), oct(expected_mode))) - - def test_chmod_existing_directory(self): - host = self.host - host.mkdir("_test dir_") - self.cleaner.add_dir("_test dir_") - # set/get mode of a directory - host.chmod("_test dir_", 0757) - self.assert_mode("_test dir_", 0757) - # set/get mode in nested directory - host.mkdir("_test dir_/nested_dir") - self.cleaner.add_dir("_test dir_/nested_dir") - # set/get mode of a directory - host.chmod("_test dir_/nested_dir", 0757) - self.assert_mode("_test dir_/nested_dir", 0757) - - def test_chmod_existing_file(self): - host = self.host - host.mkdir("_test dir_") - self.cleaner.add_dir("_test dir_") - # set/get mode on a file - file_name = host.path.join("_test dir_", "_testfile_") - self.make_file(file_name) - host.chmod(file_name, 0646) - self.assert_mode(file_name, 0646) - - def test_chmod_nonexistent_path(self): - # set/get mode of a directory - self.assertRaises(ftp_error.PermanentError, self.host.chmod, - "nonexistent", 0757) - - def test_cache_invalidation(self): - host = self.host - host.mkdir("_test dir_") - self.cleaner.add_dir("_test dir_") - # make sure the mode is in the cache - unused_stat_result = host.stat("_test dir_") - # set/get mode of a directory - host.chmod("_test dir_", 0757) - self.assert_mode("_test dir_", 0757) - # set/get mode on a file - file_name = host.path.join("_test dir_", "_testfile_") - self.make_file(file_name) - # make sure the mode is in the cache - unused_stat_result = host.stat(file_name) - host.chmod(file_name, 0646) - self.assert_mode(file_name, 0646) - - # - # other tests - # - def test_open_for_reading(self): - # test for issue #17, http://ftputil.sschwarzer.net/trac/ticket/17 - file1 = self.host.file("debian-keyring.tar.gz", 'rb') - file1.close() - # make sure that there are no problems if the connection is reused - file2 = self.host.file("debian-keyring.tar.gz", 'rb') - file2.close() - self.failUnless(file1._session is file2._session) - - def test_names_with_spaces(self): - # test if directories and files with spaces in their names - # can be used - host = self.host - self.failUnless(host.path.isdir("dir with spaces")) - self.assertEqual(host.listdir("dir with spaces"), - ['second dir', 'some file', 'some_file']) - self.failUnless(host.path.isdir("dir with spaces/second dir")) - self.failUnless(host.path.isfile("dir with spaces/some_file")) - self.failUnless(host.path.isfile("dir with spaces/some file")) + # clean up + host1.close() + host2.close() @@ -594,6 +349,5 @@ remote server. Thus, you may want to skip this test by pressing [Ctrl-C]. If the test should run, enter the login data for the remote server. You -need write access in the login directory. This test can last a few minutes -because it has to wait to test the timezone calculation. +need write access in the login directory. This test can last a few minutes. """ try: Index: test_with_statement.py =================================================================== --- _test_with_statement.py (revision 708:e0b3a3cde421) +++ (revision ) @@ -1,123 +1,0 @@ -# Copyright (C) 2008, Roger Demetrescu, Stefan Schwarzer -# All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are -# met: -# -# - Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# -# - Redistributions in binary form must reproduce the above copyright -# notice, this list of conditions and the following disclaimer in the -# documentation and/or other materials provided with the distribution. -# -# - Neither the name of the above author nor the names of the -# contributors to the software may be used to endorse or promote -# products derived from this software without specific prior written -# permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -# ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR -# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -# LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -# NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -# $Id: _test_with_statement.py 689 2007-04-16 01:07:10Z schwa $ - -from __future__ import with_statement - -import unittest - -import _test_base -import ftp_error - -from _test_ftputil import FailOnLoginSession -from _test_ftp_file import InaccessibleDirSession, ReadMockSession - - -# exception raised by client code, i. e. code using ftputil -class ClientCodeException(Exception): - pass - - -# -# test cases -# -class TestHostContextManager(unittest.TestCase): - def test_normal_operation(self): - with _test_base.ftp_host_factory() as host: - self.assertEqual(host.closed, False) - self.assertEqual(host.closed, True) - - def test_ftputil_exception(self): - try: - with _test_base.ftp_host_factory(FailOnLoginSession) as host: - pass - except ftp_error.FTPOSError: - # we arrived here, that's fine - # because the `FTPHost` object wasn't successfully constructed - # the assignment to `host` shouldn't have happened - self.failIf('host' in locals()) - else: - raise self.failureException("ftp_error.FTPOSError not raised") - - def test_client_code_exception(self): - try: - with _test_base.ftp_host_factory() as host: - self.assertEqual(host.closed, False) - raise ClientCodeException() - except ClientCodeException: - self.assertEqual(host.closed, True) - else: - raise self.failureException("ClientCodeException not raised") - - -class TestFileContextManager(unittest.TestCase): - def test_normal_operation(self): - with _test_base.ftp_host_factory(session_factory=ReadMockSession) \ - as host: - with host.file('dummy', 'r') as f: - self.assertEqual(f.closed, False) - data = f.readline() - self.assertEqual(data, 'line 1\n') - self.assertEqual(f.closed, False) - self.assertEqual(f.closed, True) - - def test_ftputil_exception(self): - with _test_base.ftp_host_factory( - session_factory=InaccessibleDirSession) as host: - try: - # this should fail since the directory isn't accessible - # by definition - with host.file('/inaccessible/new_file', 'w') as f: - pass - except ftp_error.FTPIOError: - # the file construction didn't succeed, so `f` should - # be absent from the namespace - self.failIf('f' in locals()) - else: - raise self.failureException("ftp_error.FTPIOError not raised") - - def test_client_code_exception(self): - with _test_base.ftp_host_factory(session_factory=ReadMockSession) \ - as host: - try: - with host.file('dummy', 'r') as f: - self.assertEqual(f.closed, False) - raise ClientCodeException() - except ClientCodeException: - self.assertEqual(f.closed, True) - else: - raise self.failureException("ClientCodeException not raised") - - -if __name__ == '__main__': - unittest.main() - Index: announcements.txt =================================================================== --- announcements.txt (revision 832:dba35d241916) +++ announcements.txt (revision 599:fee84e3496a6) @@ -1,385 +1,2 @@ -ftputil 2.4.2 is now available from -http://ftputil.sschwarzer.net/download . - -Changes since version 2.4.1 ---------------------------- - -- Some FTP servers seem to have problems using *any* directory - argument which contains slashes. The new default for FTP commands - now is to change into the directory before actually invoking the - command on a relative path (report and fix suggestion by Nicola - Murino). - -- Calling the method ``FTPHost.stat_cache.resize`` with an argument 0 - caused an exception. This has been fixed; a zero cache size now - of course doesn't cache anything but doesn't lead to a traceback - either. - -- The installation script ``setup.py`` didn't work with the ``--home`` - option because it still tried to install the documentation in a - system directory (report by Albrecht M�chulte). - - As a side effect, when using the *global* installation, the - documentation is no longer installed in the ftputil package - directory but in a subdirectory ``doc`` of a directory determined by - Distutils. For example, on my system (Ubuntu 9.04) the documentation - files are put into ``/usr/local/doc``. - -Upgrading is recommended. - -What is ftputil? ----------------- - -ftputil is a high-level FTP client library for the Python programming -language. ftputil implements a virtual file system for accessing FTP -servers, that is, it can generate file-like objects for remote files. -The library supports many functions similar to those in the os, -os.path and shutil modules. ftputil has convenience functions for -conditional uploads and downloads, and handles FTP clients and servers -in different timezones. - -Read the documentation at -http://ftputil.sschwarzer.net/documentation . - -License -------- - -ftputil is Open Source software, released under the revised BSD -license (see http://www.opensource.org/licenses/bsd-license.php ). - -Stefan - ----------------------------------------------------------------------- -ftputil 2.4.1 is now available from -http://ftputil.sschwarzer.net/download . - -Changes since version 2.4 -------------------------- - -Several bugs were fixed: - -- On Windows, some accesses to the stat cache caused it to become - inconsistent, which could also trigger exceptions (report and patch - by Peter Stirling). - -- In ftputil 2.4, the use of ``super`` in the exception base class - caused ftputil to fail on Python <2.5 (reported by Nicola Murino). - ftputil is supposed to run with Python 2.3+. - -- The conversion of 12-hour clock times to 24-hour clock in the MS - format parser was wrong for 12 AM and 12 PM. - -Upgrading is strongly recommended. - -What is ftputil? ----------------- - -ftputil is a high-level FTP client library for the Python programming -language. ftputil implements a virtual file system for accessing FTP -servers, that is, it can generate file-like objects for remote files. -The library supports many functions similar to those in the os, -os.path and shutil modules. ftputil has convenience functions for -conditional uploads and downloads, and handles FTP clients and servers -in different timezones. - -Read the documentation at -http://ftputil.sschwarzer.net/documentation . - -License -------- - -ftputil is Open Source software, released under the revised BSD -license (see http://www.opensource.org/licenses/bsd-license.php ). - -Stefan - ----------------------------------------------------------------------- -ftputil 2.4 is now available from -http://ftputil.sschwarzer.net/download . - -Changes since version 2.3 -------------------------- - -The ``FTPHost`` class got a new method ``chmod``, similar to -``os.chmod``, to act on remote files. Thanks go to Tom Parker for -the review. - -There's a new exception ``CommandNotImplementedError``, derived from -``PermanentError``, to denote commands not implemented by the FTP -server or disabled by its administrator. - -Using the ``xreadlines`` method of FTP file objects causes a warning -through Python's warnings framework. - -Upgrading is recommended. - -Incompatibility notice ----------------------- - -The ``xreadlines`` method will be removed in ftputil *2.5* as well as -the direct access of exception classes via the ftputil module (e. g. -``ftputil.PermanentError``). However, the deprecated access causes no -warning because that would be rather difficult to implement. - -The distribution contains a small tool find_deprecated_code.py to scan -a directory tree for the deprecated uses. Invoke the program with the -``--help`` option to see a description. - -What is ftputil? ----------------- - -ftputil is a high-level FTP client library for the Python programming -language. ftputil implements a virtual file system for accessing FTP -servers, that is, it can generate file-like objects for remote files. -The library supports many functions similar to those in the os, -os.path and shutil modules. ftputil has convenience functions for -conditional uploads and downloads, and handles FTP clients and servers -in different timezones. - -Read the documentation at -http://ftputil.sschwarzer.net/documentation . - -License -------- - -ftputil is Open Source software, released under the revised BSD -license (see http://www.opensource.org/licenses/bsd-license.php ). - -Stefan - ----------------------------------------------------------------------- -ftputil 2.3 is now available from -http://ftputil.sschwarzer.net/download . - -Changes since version 2.2.4 ---------------------------- - -ftputil has got support for the ``with`` statement which was introduced -by Python 2.5. You can now construct host and remote file objects in -``with`` statements and have them closed automatically (contributed -by Roger Demetrescu). See the documentation for examples. - -What is ftputil? ----------------- - -ftputil is a high-level FTP client library for the Python programming -language. ftputil implements a virtual file system for accessing FTP -servers, that is, it can generate file-like objects for remote files. -The library supports many functions similar to those in the os, -os.path and shutil modules. ftputil has convenience functions for -conditional uploads and downloads, and handles FTP clients and servers -in different timezones. - -Read the documentation at -http://ftputil.sschwarzer.net/documentation . - -License -------- - -ftputil is Open Source software, released under the revised BSD -license (see http://www.opensource.org/licenses/bsd-license.php ). - -Stefan - ----------------------------------------------------------------------- -ftputil 2.2.4 is now available from -http://ftputil.sschwarzer.net/download . - -Changes since version 2.2.3 ---------------------------- - -This release fixes a bug in the ``makedirs`` call (report and fix by -Richard Holden). Upgrading is recommended. - -What is ftputil? ----------------- - -ftputil is a high-level FTP client library for the Python programming -language. ftputil implements a virtual file system for accessing FTP -servers, that is, it can generate file-like objects for remote files. -The library supports many functions similar to those in the os, -os.path and shutil modules. ftputil has convenience functions for -conditional uploads and downloads, and handles FTP clients and servers -in different timezones. - -Read the documentation at -http://ftputil.sschwarzer.net/documentation . - -License -------- - -ftputil is Open Source software, released under the revised BSD -license (see http://www.opensource.org/licenses/bsd-license.php ). - -Stefan - ----------------------------------------------------------------------- -ftputil 2.2.3 is now available from -http://ftputil.sschwarzer.net/download . - -Changes since version 2.2.2 ---------------------------- - -This release fixes a bug in the ``makedirs`` call (report and fix by -Julian, whose last name I don't know ;-) ). Upgrading is recommended. - -What is ftputil? ----------------- - -ftputil is a high-level FTP client library for the Python programming -language. ftputil implements a virtual file system for accessing FTP -servers, that is, it can generate file-like objects for remote files. -The library supports many functions similar to those in the os, -os.path and shutil modules. ftputil has convenience functions for -conditional uploads and downloads, and handles FTP clients and servers -in different timezones. - -Read the documentation at -http://ftputil.sschwarzer.net/documentation . - -License -------- - -ftputil is Open Source software, released under the revised BSD -license (see http://www.opensource.org/licenses/bsd-license.php ). - -Stefan - ----------------------------------------------------------------------- -ftputil 2.2.2 is now available from -http://ftputil.sschwarzer.net/download . - -Changes since version 2.2.1 ---------------------------- - -This bugfix release handles whitespace in path names more reliably -(thanks to Johannes Str�rg). Upgrading is recommended. - -What is ftputil? ----------------- - -ftputil is a high-level FTP client library for the Python programming -language. ftputil implements a virtual file system for accessing FTP -servers, that is, it can generate file-like objects for remote files. -The library supports many functions similar to those in the os, -os.path and shutil modules. ftputil has convenience functions for -conditional uploads and downloads, and handles FTP clients and servers -in different timezones. - -Read the documentation at -http://ftputil.sschwarzer.net/documentation . - -License -------- - -ftputil is Open Source software, released under the revised BSD -license (see http://www.opensource.org/licenses/bsd-license.php ). - -Stefan - ----------------------------------------------------------------------- -ftputil 2.2.1 is now available from -http://ftputil.sschwarzer.net/download . - -Changes since version 2.2 -------------------------- - -This bugfix release checks (and ignores) status code 451 when FTPFiles -are closed (thanks go to Alexander Holyapin). Upgrading is recommended. - -What is ftputil? ----------------- - -ftputil is a high-level FTP client library for the Python programming -language. ftputil implements a virtual file system for accessing FTP -servers, that is, it can generate file-like objects for remote files. -The library supports many functions similar to those in the os, -os.path and shutil modules. ftputil has convenience functions for -conditional uploads and downloads, and handles FTP clients and servers -in different timezones. - -Read the documentation at -http://ftputil.sschwarzer.net/trac/wiki/Documentation . - -License -------- - -ftputil is Open Source software, released under the revised BSD -license (see http://www.opensource.org/licenses/bsd-license.php ). - -Stefan - ----------------------------------------------------------------------- -ftputil 2.2 is now available from -http://ftputil.sschwarzer.net/download . - -Changes since version 2.1 -------------------------- - -- Results of stat calls (also indirect calls, i. e. listdir, - isdir/isfile/islink, exists, getmtime etc.) are now cached and - reused. This results in remarkable speedups for many use cases. - Thanks to Evan Prodromou for his permission to add his lrucache - module under ftputil's license. - -- The current directory is also locally cached, resulting in further - speedups. - -- It's now possible to write and plug in custom parsers for directory - formats which ftputil doesn't support natively. - -- File-like objects generated via ``FTPHost.file`` now support the - iterator protocol (for line in some_file: ...). - -- The documentation has been updated accordingly. Read it under - http://ftputil.sschwarzer.net/trac/wiki/Documentation . - -Possible incompatibilities: - -- This release requires at least Python 2.3. (Previous releases - worked with Python versions from 2.1 up.) - -- The method ``FTPHost.set_directory_format`` has been removed, - since the directory format (Unix or MS) is set automatically. (The - new method ``set_parser`` is a different animal since it takes - a parser object to parse "foreign" formats, not a string.) - -What is ftputil? ----------------- - -ftputil is a high-level FTP client library for the Python programming -language. ftputil implements a virtual file system for accessing FTP -servers, that is, it can generate file-like objects for remote files. -The library supports many functions similar to those in the os, -os.path and shutil modules. ftputil has convenience functions for -conditional uploads and downloads, and handles FTP clients and servers -in different timezones. - -License -------- - -ftputil 2.2 is Open Source software, released under the revised BSD -license (see http://www.opensource.org/licenses/bsd-license.php ). - -Stefan - ----------------------------------------------------------------------- -The second beta version of ftputil 2.2 is available. You can download -it from http://ftputil.sschwarzer.net/download . - -With respect to the first beta release, it's now possible to write -and plug in custom parsers for FTP directory formats that ftputil -doesn't know natively. The documentation has been updated accordingly. - -The documentation for this release is online at -http://ftputil.sschwarzer.net/trac/wiki/Documentation#Documentationforftputil2.2b2 , -so you can read about the extensions in more detail. - -Please download and test the release. Do you miss something which -should be in this release? Are there any bugs? - -Stefan - ----------------------------------------------------------------------- The first beta version of ftputil 2.2 is available. You can download it from http://ftputil.sschwarzer.net/download . Index: ebian/custom/changelog =================================================================== --- debian/custom/changelog (revision 799:c08941e6112b) +++ (revision ) @@ -1,6 +1,0 @@ -ftputil (2.4.1) unstable; urgency=low - - * Initial release - - -- Stefan Schwarzer Thu, 01 Jan 2009 18:20:37 +0100 - Index: ebian/custom/control =================================================================== --- debian/custom/control (revision 759:c94855dd8063) +++ (revision ) @@ -1,20 +1,0 @@ -Source: ftputil -Section: python -Priority: extra -Maintainer: Stefan Schwarzer -Build-Depends: cdbs (>= 0.4.49), python-central (>= 0.6), debhelper (>= 5.0.38) -Standards-Version: 3.7.3 -XS-Python-Version: >= 2.3 -Homepage: http://ftputil.sschwarzer.net - -Package: python-ftputil -Architecture: all -XB-Python-Version: ${python:Versions} -Depends: ${python:Depends}, ${misc:Depends} -Description: High-level FTP client library - ftputil is a high-level FTP client library for the Python programming - language. ftputil implements a virtual file system for accessing FTP servers, - that is, it can generate file-like objects for remote files. The library - supports many functions similar to those in the os, os.path and shutil - modules. ftputil has convenience functions for conditional uploads and - downloads, and handles FTP clients and servers in different timezones. Index: ebian/custom/copyright =================================================================== --- debian/custom/copyright (revision 799:c08941e6112b) +++ (revision ) @@ -1,41 +1,0 @@ -This package was debianized by Stefan Schwarzer on -Thu, 01 Jan 2009 17:19:16 +0100. - -It was downloaded from http://ftputil.sschwarzer.net/download - -Upstream Authors: - - Stefan Schwarzer - Evan Prodromou - Roger Demetrescu - -Copyright: - - Copyright (C) 2002-2009 Stefan Schwarzer - (C) 2004 Evan Prodromou - (C) 2008 Roger Demetrescu - -License: - - Redistribution and use in source and binary forms, with or without - modification, are permitted under the terms of the BSD License. - - THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND - ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE - FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL - DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS - OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) - HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT - LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY - OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF - SUCH DAMAGE. - -On Debian systems, the complete text of the BSD License can be -found in `/usr/share/common-licenses/BSD'. - - -The Debian packaging is (C) 2009, Stefan Schwarzer -and is licensed under the same license as ftputil. - Index: ebian/custom/rules =================================================================== --- debian/custom/rules (revision 759:c94855dd8063) +++ (revision ) @@ -1,10 +1,0 @@ -#!/usr/bin/make -f - -DEB_PYTHON_SYSTEM=pycentral -DEB_COMPRESS_EXCLUDE := .py - -include /usr/share/cdbs/1/rules/debhelper.mk -include /usr/share/cdbs/1/class/python-distutils.mk - - -# Add here any variable or target overrides you need. Index: efault.css =================================================================== --- default.css (revision 614:5e63dda13a86) +++ (revision ) @@ -1,263 +1,0 @@ -/* -:Author: David Goodger -:Contact: goodger@users.sourceforge.net -:Date: $Date: 2005-05-26 12:51:39 +0200 (Thu, 26 May 2005) $ -:Version: $Revision: 3368 $ -:Copyright: This stylesheet has been placed in the public domain. - -Default cascading style sheet for the HTML output of Docutils. -*/ - -/* "! important" is used here to override other ``margin-top`` and - ``margin-bottom`` styles that are later in the stylesheet or - more specific. See http://www.w3.org/TR/CSS1#the-cascade */ -.first { - margin-top: 0 ! important } - -.last, .with-subtitle { - margin-bottom: 0 ! important } - -.hidden { - display: none } - -a.toc-backref { - text-decoration: none ; - color: black } - -blockquote.epigraph { - margin: 2em 5em ; } - -dl.docutils dd { - margin-bottom: 0.5em } - -/* Uncomment (and remove this text!) to get bold-faced definition list terms -dl.docutils dt { - font-weight: bold } -*/ - -div.abstract { - margin: 2em 5em } - -div.abstract p.topic-title { - font-weight: bold ; - text-align: center } - -div.admonition, div.attention, div.caution, div.danger, div.error, -div.hint, div.important, div.note, div.tip, div.warning { - margin: 2em ; - border: medium outset ; - padding: 1em } - -div.admonition p.admonition-title, div.hint p.admonition-title, -div.important p.admonition-title, div.note p.admonition-title, -div.tip p.admonition-title { - font-weight: bold ; - font-family: sans-serif } - -div.attention p.admonition-title, div.caution p.admonition-title, -div.danger p.admonition-title, div.error p.admonition-title, -div.warning p.admonition-title { - color: red ; - font-weight: bold ; - font-family: sans-serif } - -/* Uncomment (and remove this text!) to get reduced vertical space in - compound paragraphs. -div.compound .compound-first, div.compound .compound-middle { - margin-bottom: 0.5em } - -div.compound .compound-last, div.compound .compound-middle { - margin-top: 0.5em } -*/ - -div.dedication { - margin: 2em 5em ; - text-align: center ; - font-style: italic } - -div.dedication p.topic-title { - font-weight: bold ; - font-style: normal } - -div.figure { - margin-left: 2em } - -div.footer, div.header { - font-size: smaller } - -div.line-block { - display: block ; - margin-top: 1em ; - margin-bottom: 1em } - -div.line-block div.line-block { - margin-top: 0 ; - margin-bottom: 0 ; - margin-left: 1.5em } - -div.sidebar { - margin-left: 1em ; - border: medium outset ; - padding: 1em ; - background-color: #ffffee ; - width: 40% ; - float: right ; - clear: right } - -div.sidebar p.rubric { - font-family: sans-serif ; - font-size: medium } - -div.system-messages { - margin: 5em } - -div.system-messages h1 { - color: red } - -div.system-message { - border: medium outset ; - padding: 1em } - -div.system-message p.system-message-title { - color: red ; - font-weight: bold } - -div.topic { - margin: 2em } - -h1.section-subtitle, h2.section-subtitle, h3.section-subtitle, -h4.section-subtitle, h5.section-subtitle, h6.section-subtitle { - margin-top: 0.4em } - -h1.title { - text-align: center } - -h2.subtitle { - text-align: center } - -hr.docutils { - width: 75% } - -ol.simple, ul.simple { - margin-bottom: 1em } - -ol.arabic { - list-style: decimal } - -ol.loweralpha { - list-style: lower-alpha } - -ol.upperalpha { - list-style: upper-alpha } - -ol.lowerroman { - list-style: lower-roman } - -ol.upperroman { - list-style: upper-roman } - -p.attribution { - text-align: right ; - margin-left: 50% } - -p.caption { - font-style: italic } - -p.credits { - font-style: italic ; - font-size: smaller } - -p.label { - white-space: nowrap } - -p.rubric { - font-weight: bold ; - font-size: larger ; - color: maroon ; - text-align: center } - -p.sidebar-title { - font-family: sans-serif ; - font-weight: bold ; - font-size: larger } - -p.sidebar-subtitle { - font-family: sans-serif ; - font-weight: bold } - -p.topic-title { - font-weight: bold } - -pre.address { - margin-bottom: 0 ; - margin-top: 0 ; - font-family: serif ; - font-size: 100% } - -pre.line-block { - font-family: serif ; - font-size: 100% } - -pre.literal-block, pre.doctest-block { - margin-left: 2em ; - margin-right: 2em ; - background-color: #eeeeee } - -span.classifier { - font-family: sans-serif ; - font-style: oblique } - -span.classifier-delimiter { - font-family: sans-serif ; - font-weight: bold } - -span.interpreted { - font-family: sans-serif } - -span.option { - white-space: nowrap } - -span.pre { - white-space: pre } - -span.problematic { - color: red } - -span.section-subtitle { - /* font-size relative to parent ( element) */ - font-size: 80% } - -table.citation { - border-left: solid thin gray } - -table.docinfo { - margin: 2em 4em } - -table.docutils { - margin-top: 0.5em ; - margin-bottom: 0.5em } - -table.footnote { - border-left: solid thin black } - -table.docutils td, table.docutils th, -table.docinfo td, table.docinfo th { - padding-left: 0.5em ; - padding-right: 0.5em ; - vertical-align: top } - -table.docutils th.field-name, table.docinfo th.docinfo-name { - font-weight: bold ; - text-align: left ; - white-space: nowrap ; - padding-left: 0 } - -h1 tt.docutils, h2 tt.docutils, h3 tt.docutils, -h4 tt.docutils, h5 tt.docutils, h6 tt.docutils { - font-size: 100% } - -tt.docutils { - background-color: #eeeeee } - -ul.auto-toc { - list-style-type: none } Index: ind_deprecated_code.py =================================================================== --- find_deprecated_code.py (revision 745:94114c042f28) +++ (revision ) @@ -1,148 +1,0 @@ -#! /usr/bin/env python -# Copyright (C) 2008, Stefan Schwarzer -# All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are -# met: -# -# - Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# -# - Redistributions in binary form must reproduce the above copyright -# notice, this list of conditions and the following disclaimer in the -# documentation and/or other materials provided with the distribution. -# -# - Neither the name of the above author nor the names of the -# contributors to the software may be used to endorse or promote -# products derived from this software without specific prior written -# permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -# ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR -# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -# LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -# NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -# $Id: $ - -# pylint: disable-msg=W0622 - -"""\ -This script scans a directory tree for files which contain code which -is deprecated in ftputil %s and above (and even much longer). The -script uses simple heuristics, so it may miss occurences of deprecated -usage or print some inappropriate lines of your files. - -Usage: %s start_dir - -where start_dir is the starting directory which will be scanned -recursively for offending code. - -Currently, these deprecated features are examined: - -- You should no longer use the exceptions via the ftputil module but - via the ftp_error module. So, for example, instead of - ftputil.PermanentError write ftp_error.PermanentError. - -- Don't use the xreadlines method of FTP file objects (as returned by - FTPHost.file = FTPHost.open). Instead use - - for line in ftp_host.open(path): - ... -""" - -import ftputil_version - -import os -import re -import sys - -__doc__ = __doc__ % (ftputil_version.__version__, os.path.basename(sys.argv[0])) - -deprecated_features = [ - ("Possible use(s) of FTP exceptions via ftputil module", - re.compile(r"\bftputil\s*?\.\s*?[A-Za-z]+Error\b"), {}), - ("Possible use(s) of xreadline method of FTP file objects", - re.compile(r"\.\s*?xreadlines\b"), {}), -] - -def scan_file(file_name): - """ - Scan a file with name `file_name` for code deprecated in - ftputil usage and collect the offending data in the data - structure `deprecated_features`. - """ - fobj = open(file_name) - try: - for index, line in enumerate(fobj): - # `title` isn't used here - # pylint: disable-msg=W0612 - for title, regex, positions in deprecated_features: - if regex.search(line): - positions.setdefault(file_name, []) - positions[file_name].append((index+1, line.rstrip())) - finally: - fobj.close() - -def print_results(): - """ - Print statistics of deprecated code after the directory has been - scanned. - """ - last_title = "" - # `regex` isn't used here - # pylint: disable-msg=W0612 - for title, regex, positions in deprecated_features: - if title != last_title: - print - print title, "..." - print - last_title = title - if not positions: - print " no deprecated code found" - continue - file_names = positions.keys() - file_names.sort() - for file_name in file_names: - print file_name - for line_number, line in positions[file_name]: - print "%5d: %s" % (line_number, line) - print - print "If possible, check your code also by other means." - -def main(start_dir): - """ - Scan a directory tree starting at `start_dir` and print uses - of deprecated features, if any were found. - """ - # `dir_names` isn't used here - # pylint: disable-msg=W0612 - for dir_path, dir_names, file_names in os.walk(start_dir): - for file_name in file_names: - abs_name = os.path.abspath(os.path.join(dir_path, file_name)) - if file_name.endswith(".py"): - scan_file(abs_name) - print_results() - - -if __name__ == '__main__': - if len(sys.argv) == 2: - if sys.argv[1] in ("-h", "--help"): - print __doc__ - sys.exit(0) - start_dir = sys.argv[1] - if not os.path.isdir(start_dir): - print >> sys.stderr, "Directory %s not found." % start_dir - sys.exit() - else: - print >> sys.stderr, "Usage: %s start_dir" % sys.argv[0] - sys.exit() - main(start_dir) - Index: ftp_error.py =================================================================== --- ftp_error.py (revision 791:06f681345c6e) +++ ftp_error.py (revision 499:7e61a5b0e3e7) @@ -1,3 +1,3 @@ -# Copyright (C) 2003-2009, Stefan Schwarzer +# Copyright (C) 2003-2006, Stefan Schwarzer # All rights reserved. # @@ -36,40 +36,15 @@ # $Id$ -# "Too many ancestors" -# pylint: disable-msg = R0901 - import ftplib import sys -import warnings import ftputil_version -class FTPError(Exception): +class FTPError: """General error class.""" - - def __init__(self, *args): - # `ftplib.Error` doesn't have a `__subclasses__` _method_ but a - # static method, so my use of `ftplib.Error.__subclasses__` in - # my opinion is valid - # pylint: disable-msg = e1101 - # contrary to what `ftplib`'s documentation says, `all_errors` - # does _not_ contain the subclasses, so I explicitly add them - if args and (args[0].__class__ in ftplib.all_errors or - issubclass(args[0].__class__, ftplib.Error)): - warnings.warn(("Passing exception objects into the FTPError " - "constructor is deprecated and will be disabled in ftputil 2.6"), - DeprecationWarning, stacklevel=2) - try: - # works only for new style-classes (Python 2.5+) - super(FTPError, self).__init__(*args) - except TypeError: - # fallback to old approach - Exception.__init__(self, *args) - # don't use `args[0]` because `args` may be empty - if args: - self.strerror = self.args[0] - else: - self.strerror = "" + def __init__(self, ftp_exception): + self.args = (ftp_exception,) + self.strerror = str(ftp_exception) try: self.errno = int(self.strerror[:3]) @@ -84,50 +59,14 @@ # internal errors are those that have more to do with the inner # workings of ftputil than with errors on the server side -class InternalError(FTPError): - """Internal error.""" - pass +class InternalError(FTPError): pass +class RootDirError(InternalError): pass +class InaccessibleLoginDirError(InternalError): pass +class TimeShiftError(InternalError): pass +class ParserError(InternalError): pass +class KeepAliveError(InternalError): pass -class RootDirError(InternalError): - """Raised for generic stat calls on the remote root directory.""" - pass - -class InaccessibleLoginDirError(InternalError): - """May be raised if the login directory isn't accessible.""" - pass - -class TimeShiftError(InternalError): - """Raised for invalid time shift values.""" - pass - -class ParserError(InternalError): - """Raised if a line of a remote directory can't be parsed.""" - pass - -# currently not used -class KeepAliveError(InternalError): - """Raised if the keep-alive feature failed.""" - pass - -class FTPOSError(FTPError, OSError): - """Generic FTP error related to `OSError`.""" - pass - -class TemporaryError(FTPOSError): - """Raised for temporary FTP errors (4xx).""" - pass - -class PermanentError(FTPOSError): - """Raised for permanent FTP errors (5xx).""" - pass - -class CommandNotImplementedError(PermanentError): - """Raised if the server doesn't implement a certain feature (502).""" - pass - -# currently not used -class SyncError(PermanentError): - """Raised for problems specific to syncing directories.""" - pass - +class FTPOSError(FTPError, OSError): pass +class TemporaryError(FTPOSError): pass +class PermanentError(FTPOSError): pass #XXX Do you know better names for `_try_with_oserror` and @@ -139,24 +78,15 @@ derived classes. """ - # use `*exc.args` instead of `str(args)` because args might be - # a unicode string with non-ascii characters try: return callee(*args, **kwargs) - except ftplib.error_temp, exc: - raise TemporaryError(*exc.args) - except ftplib.error_perm, exc: - # if `exc.args` is present, assume it's a byte or unicode string - if exc.args and exc.args[0].startswith("502"): - raise CommandNotImplementedError(*exc.args) - else: - raise PermanentError(*exc.args) + except ftplib.error_temp, obj: + raise TemporaryError(obj) + except ftplib.error_perm, obj: + raise PermanentError(obj) except ftplib.all_errors: - exc = sys.exc_info()[1] - raise FTPOSError(*exc.args) + ftp_error = sys.exc_info()[1] + raise FTPOSError(ftp_error) -class FTPIOError(FTPError, IOError): - """Generic FTP error related to `IOError`.""" - pass - +class FTPIOError(FTPError, IOError): pass def _try_with_ioerror(callee, *args, **kwargs): @@ -168,7 +98,5 @@ return callee(*args, **kwargs) except ftplib.all_errors: - exc = sys.exc_info()[1] - # use `*exc.args` instead of `str(args)` because args might be - # a unicode string with non-ascii characters - raise FTPIOError(*exc.args) + ftp_error = sys.exc_info()[1] + raise FTPIOError(ftp_error) Index: ftp_file.py =================================================================== --- ftp_file.py (revision 729:0d93a978ad85) +++ ftp_file.py (revision 586:f0832919e4bc) @@ -1,3 +1,3 @@ -# Copyright (C) 2003-2008, Stefan Schwarzer +# Copyright (C) 2003-2006, Stefan Schwarzer # All rights reserved. # @@ -36,6 +36,4 @@ # $Id$ -import warnings - import ftp_error @@ -50,25 +48,14 @@ # "\n" would fail, if "\r" (without "\n") occured at the end of the # string `text` -def _crlf_to_python_linesep(text): - """ - Return `text` with ASCII line endings (CR/LF) converted to - Python's internal representation (LF). - """ - return text.replace('\r', '') +_crlf_to_python_linesep = lambda text: text.replace('\r', '') # converter for Python line ends into `\r\n` -def _python_to_crlf_linesep(text): - """ - Return `text` with Python's internal line ending representation - (LF) converted to ASCII line endings (CR/LF). - """ - return text.replace('\n', '\r\n') +_python_to_crlf_linesep = lambda text: text.replace('\n', '\r\n') # helper class for xreadline protocol for ASCII transfers #XXX maybe we can use the `xreadlines` module instead of this? -class _XReadlines(object): +class _XReadlines: """Represents `xreadline` objects for ASCII transfers.""" - def __init__(self, ftp_file): self._ftp_file = ftp_file @@ -88,5 +75,5 @@ -class _FTPFile(object): +class _FTPFile: """ Represents a file-like object connected to an FTP host. File and @@ -94,5 +81,4 @@ requested. """ - def __init__(self, host): """Construct the file(-like) object.""" @@ -101,9 +87,4 @@ # the file is closed yet self.closed = True - # overwritten later in `_open` - self._bin_mode = None - self._conn = None - self._read_mode = None - self._fo = None def _open(self, path, mode): @@ -202,6 +183,4 @@ separator conversion support. """ - warnings.warn(("FTPFile.xreadlines is deprecated and will be removed " - "in ftputil 2.5"), DeprecationWarning, stacklevel=2) if self._bin_mode: return self._fo.xreadlines() @@ -242,19 +221,4 @@ # - # context manager methods - # - def __enter__(self): - # return `self`, so it can be accessed as the variable - # component of the `with` statement. - return self - - def __exit__(self, exc_type, exc_val, exc_tb): - # we don't need the `exc_*` arguments here - # pylint: disable-msg=W0613 - self.close() - # be explicit - return False - - # # other attributes # @@ -272,7 +236,5 @@ def close(self): """Close the `FTPFile`.""" - if self.closed: - return - try: + if not self.closed: self._fo.close() ftp_error._try_with_ioerror(self._conn.close) @@ -283,11 +245,6 @@ # http://ftputil.sschwarzer.net/trac/ticket/17 error_code = str(exception).split()[0] - if error_code not in ("426", "450", "451"): + if error_code not in ("426", "450"): raise - finally: - # if something went wrong before, the file is probably - # defunct and subsequent calls to `close` won't help - # either, so we consider the file closed for practical - # purposes self.closed = True Index: ftp_path.py =================================================================== --- ftp_path.py (revision 714:5353ec9a050e) +++ ftp_path.py (revision 539:a807c01d17f4) @@ -1,3 +1,3 @@ -# Copyright (C) 2003-2008, Stefan Schwarzer +# Copyright (C) 2003-2006, Stefan Schwarzer # All rights reserved. # @@ -42,5 +42,5 @@ -class _Path(object): +class _Path: """ Support class resembling `os.path`, accessible from the `FTPHost` @@ -49,5 +49,4 @@ Hint: substitute `os` with the `FTPHost` object. """ - def __init__(self, host): self._host = host @@ -72,5 +71,4 @@ def exists(self, path): - """Return true if the path exists.""" try: lstat_result = self._host.lstat( @@ -81,22 +79,7 @@ def getmtime(self, path): - """ - Return the timestamp for the last modification for `path` - as a float. - - This will raise `PermanentError` if the path doesn't exist, - but maybe other exceptions depending on the state of the - server (e. g. timeout). - """ return self._host.stat(path).st_mtime def getsize(self, path): - """ - Return the size of the `path` item as an integer. - - This will raise `PermanentError` if the path doesn't exist, - but maybe raise other exceptions depending on the state of the - server (e. g. timeout). - """ return self._host.stat(path).st_size @@ -112,10 +95,4 @@ def isfile(self, path): - """ - Return true if the `path` exists and corresponds to a regular - file (no link). - - A non-existing path does _not_ cause a `PermanentError`. - """ # workaround if we can't go up from the current directory if path == self._host.getcwd(): @@ -132,10 +109,4 @@ def isdir(self, path): - """ - Return true if the `path` exists and corresponds to a - directory (no link). - - A non-existing path does _not_ cause a `PermanentError`. - """ # workaround if we can't go up from the current directory if path == self._host.getcwd(): @@ -152,9 +123,4 @@ def islink(self, path): - """ - Return true if the `path` exists and is a link. - - A non-existing path does _not_ cause a `PermanentError`. - """ try: lstat_result = self._host.lstat( @@ -195,8 +161,8 @@ name = self.join(top, name) try: - stat_result = self._host.lstat(name) + st = self._host.lstat(name) except OSError: continue - if stat.S_ISDIR(stat_result[stat.ST_MODE]): + if stat.S_ISDIR(st[stat.ST_MODE]): self.walk(name, func, arg) Index: ftp_stat.py =================================================================== --- ftp_stat.py (revision 821:27b6f88cee1e) +++ ftp_stat.py (revision 603:83dda12c086f) @@ -1,3 +1,3 @@ -# Copyright (C) 2002-2008, Stefan Schwarzer +# Copyright (C) 2002-2006, Stefan Schwarzer # All rights reserved. # @@ -36,6 +36,6 @@ # $Id$ -import re import stat +import sys import time @@ -44,10 +44,9 @@ -class StatResult(tuple): +class _StatResult(tuple): """ Support class resembling a tuple like that returned from `os.(l)stat`. """ - _index_mapping = { 'st_mode': 0, 'st_ino': 1, 'st_dev': 2, 'st_nlink': 3, @@ -55,18 +54,9 @@ 'st_mtime': 8, 'st_ctime': 9, '_st_name': 10, '_st_target': 11} - def __init__(self, sequence): - # Don't call `__init__` via `super`. Construction from a - # sequence is implicitly handled by `tuple.__new__`, not - # `tuple.__init__`. As a by-product, this avoids a - # `DeprecationWarning` in Python 2.6+ - # these may be overwritten in a `Parser.parse_line` method - self._st_name = "" - self._st_target = None - def __getattr__(self, attr_name): if self._index_mapping.has_key(attr_name): - return self[self._index_mapping[attr_name]] - else: - raise AttributeError("'StatResult' object has no attribute '%s'" % + return self[ self._index_mapping[attr_name] ] + else: + raise AttributeError("'_Stat' object has no attribute '%s'" % attr_name) @@ -74,10 +64,32 @@ # FTP directory parsers # -class Parser(object): - """ - Represent a parser for directory lines. Parsers for specific - directory formats inherit from this class. - """ - +class _DirectoryParser: + def parse_line(self, line, time_shift=0.0): + """ + Return a `_StatResult` object as derived from the string + `line`. The parser code to use depends on the directory format + the FTP server delivers (also see examples at end of file). + + For the definition of `time_shift` see the docstring of + `FTPHost.set_time_shift` in `ftputil.py`. Not all parsers + use the `time_shift` parameter. + """ + raise NotImplementedError("must be defined by subclass") + + def parse_lines(self, lines, time_shift=0.0): + """ + Return a list of `_StatResult` objects with one `_StatResult` + object per line in the list `lines`. The order of the returned + list corresponds to the order in the `lines` argument. + + For the definition of `time_shift` see the docstring of + `FTPHost.set_time_shift` in `ftputil.py`. Not all parsers + use the `time_shift` parameter. + """ + return [self.parse_line(line, time_shift) for line in lines] + + +class _UnixDirectoryParser(_DirectoryParser): + """`_DirectoryParser` class for Unix-specific directory format.""" # map month abbreviations to month numbers _month_numbers = { @@ -85,163 +97,4 @@ 'may': 5, 'jun': 6, 'jul': 7, 'aug': 8, 'sep': 9, 'oct': 10, 'nov': 11, 'dec': 12} - - _total_regex = re.compile(r"^total\s+\d+") - - def ignores_line(self, line): - """ - Return a true value if the line should be ignored, i. e. is - assumed to _not_ contain actual directory/file/link data. - A typical example are summary lines like "total 23" which - are emitted by some FTP servers. - - If the line should be used to extract stat data from it, - return a false value. - """ - # either a `_SRE_Match` instance or `None` - match = self._total_regex.search(line) - return bool(match) - - def parse_line(self, line, time_shift=0.0): - """ - Return a `StatResult` object as derived from the string - `line`. The parser code to use depends on the directory format - the FTP server delivers (also see examples at end of file). - - If the given text line can't be parsed, raise a `ParserError`. - - For the definition of `time_shift` see the docstring of - `FTPHost.set_time_shift` in `ftputil.py`. Not all parsers - use the `time_shift` parameter. - """ - raise NotImplementedError("must be defined by subclass") - - # - # helper methods for parts of a directory listing line - # - def parse_unix_mode(self, mode_string): - """ - Return an integer from the `mode_string`, compatible with - the `st_mode` value in stat results. Such a mode string - may look like "drwxr-xr-x". - - If the mode string can't be parsed, raise an - `ftp_error.ParserError`. - """ - st_mode = 0 - if len(mode_string) != 10: - raise ftp_error.ParserError("invalid mode string '%s'" % - mode_string) - for bit in mode_string[1:10]: - bit = (bit != '-') - st_mode = (st_mode << 1) + bit - if mode_string[3] == 's': - st_mode = st_mode | stat.S_ISUID - if mode_string[6] == 's': - st_mode = st_mode | stat.S_ISGID - file_type_to_mode = {'d': stat.S_IFDIR, 'l': stat.S_IFLNK, - 'c': stat.S_IFCHR, '-': stat.S_IFREG} - file_type = mode_string[0] - if file_type in file_type_to_mode: - st_mode = st_mode | file_type_to_mode[file_type] - else: - raise ftp_error.ParserError( - "unknown file type character '%s'" % file_type) - return st_mode - - def parse_unix_time(self, month_abbreviation, day, year_or_time, - time_shift): - """ - Return a floating point number, like from `time.mktime`, by - parsing the string arguments `month_abbreviation`, `day` and - `year_or_time`. The parameter `time_shift` is the difference - "time on server" - "time on client" and is available as the - `time_shift` parameter in the `parse_line` interface. - - Times in Unix-style directory listings typically have one of - these formats: - - - "Nov 23 02:33" (month name, day of month, time) - - - "May 26 2005" (month name, day of month, year) - - If this method can not make sense of the given arguments, it - raises an `ftp_error.ParserError`. - """ - try: - month = self._month_numbers[month_abbreviation.lower()] - except KeyError: - raise ftp_error.ParserError("invalid month name '%s'" % month) - day = int(day) - if ":" not in year_or_time: - # `year_or_time` is really a year - year, hour, minute = int(year_or_time), 0, 0 - st_mtime = time.mktime( (year, month, day, - hour, minute, 0, 0, 0, -1) ) - else: - # `year_or_time` is a time hh:mm - hour, minute = year_or_time.split(':') - year, hour, minute = None, int(hour), int(minute) - # try the current year - year = time.localtime()[0] - st_mtime = time.mktime( (year, month, day, - hour, minute, 0, 0, 0, -1) ) - # rhs of comparison: transform client time to server time - # (as on the lhs), so both can be compared with respect - # to the set time shift (see the definition of the time - # shift in `FTPHost.set_time_shift`'s docstring); the - # last addend allows for small deviations between the - # supposed (rounded) and the actual time shift - # #XXX the downside of this "correction" is that there is - # a one-minute time interval exactly one year ago that - # may cause that datetime to be recognized as the current - # datetime, but after all the datetime from the server - # can only be exact up to a minute - if st_mtime > time.time() + time_shift + 60.0: - # if it's in the future, use previous year - st_mtime = time.mktime( (year-1, month, day, - hour, minute, 0, 0, 0, -1) ) - return st_mtime - - def parse_ms_time(self, date, time_, time_shift): - """ - Return a floating point number, like from `time.mktime`, by - parsing the string arguments `date` and `time_`. The parameter - `time_shift` is the difference - - "time on server" - "time on client" - - and can be set as the `time_shift` parameter in the - `parse_line` interface. - - Times in MS-style directory listings typically have the - format "10-23-01 03:25PM" (month-day_of_month-two_digit_year, - hour:minute, am/pm). - - If this method can not make sense of the given arguments, it - raises an `ftp_error.ParserError`. - """ - # don't complain about unused `time_shift` argument - # pylint: disable-msg=W0613 - try: - month, day, year = [int(part) for part in date.split('-')] - if year >= 70: - year = 1900 + year - else: - year = 2000 + year - hour, minute, am_pm = time_[0:2], time_[3:5], time_[5] - hour, minute = int(hour), int(minute) - except (ValueError, IndexError): - raise ftp_error.ParserError("invalid time string '%s'" % time_) - if am_pm == 'A' and hour == 12: - hour = 0 - if am_pm == 'P' and hour != 12: - hour = hour + 12 - st_mtime = time.mktime( (year, month, day, - hour, minute, 0, 0, 0, -1) ) - return st_mtime - - -class UnixParser(Parser): - """`Parser` class for Unix-specific directory format.""" def _split_line(self, line): @@ -277,5 +130,5 @@ def parse_line(self, line, time_shift=0.0): """ - Return a `StatResult` instance corresponding to the given + Return a `_StatResult` instance corresponding to the given text line. The `time_shift` value is needed to determine to which year a datetime without an explicit year belongs. @@ -283,8 +136,25 @@ If the line can't be parsed, raise a `ParserError`. """ - mode_string, nlink, user, group, size, month, day, \ + metadata, nlink, user, group, size, month, day, \ year_or_time, name = self._split_line(line) # st_mode - st_mode = self.parse_unix_mode(mode_string) + st_mode = 0 + if len(metadata) != 10: + raise ftp_error.ParserError("invalid metadata '%s'" % metadata) + for bit in metadata[1:10]: + bit = (bit != '-') + st_mode = (st_mode << 1) + bit + if metadata[3] == 's': + st_mode = st_mode | stat.S_ISUID + if metadata[6] == 's': + st_mode = st_mode | stat.S_ISGID + char_to_mode = {'d': stat.S_IFDIR, 'l': stat.S_IFLNK, + 'c': stat.S_IFCHR, '-': stat.S_IFREG} + file_type = metadata[0] + if char_to_mode.has_key(file_type): + st_mode = st_mode | char_to_mode[file_type] + else: + raise ftp_error.ParserError( + "unknown file type character '%s'" % file_type) # st_ino, st_dev, st_nlink, st_uid, st_gid, st_size, st_atime st_ino = None @@ -296,13 +166,45 @@ st_atime = None # st_mtime - st_mtime = self.parse_unix_time(month, day, year_or_time, time_shift) + try: + month = self._month_numbers[month.lower()] + except KeyError: + raise ftp_error.ParserError("invalid month name '%s'" % month) + day = int(day) + if year_or_time.find(':') == -1: + # `year_or_time` is really a year + year, hour, minute = int(year_or_time), 0, 0 + st_mtime = time.mktime( (year, month, day, hour, + minute, 0, 0, 0, -1) ) + else: + # `year_or_time` is a time hh:mm + hour, minute = year_or_time.split(':') + year, hour, minute = None, int(hour), int(minute) + # try the current year + year = time.localtime()[0] + st_mtime = time.mktime( (year, month, day, hour, + minute, 0, 0, 0, -1) ) + # rhs of comparison: transform client time to server time + # (as on the lhs), so both can be compared with respect + # to the set time shift (see the definition of the time + # shift in `FTPHost.set_time_shift`'s docstring); the + # last addend allows for small deviations between the + # supposed (rounded) and the actual time shift + # #XXX the downside of this "correction" is that there is + # a one-minute time interval excatly one year ago that + # may cause that datetime to be recognized as the current + # datetime, but after all the datetime from the server + # can only be exact up to a minute + if st_mtime > time.time() + time_shift + 60.0: + # if it's in the future, use previous year + st_mtime = time.mktime( (year-1, month, day, + hour, minute, 0, 0, 0, -1) ) # st_ctime st_ctime = None # st_name - if " -> " in name: + if name.find(' -> ') != -1: st_name, st_target = name.split(' -> ') else: st_name, st_target = name, None - stat_result = StatResult( + stat_result = _StatResult( (st_mode, st_ino, st_dev, st_nlink, st_uid, st_gid, st_size, st_atime, st_mtime, st_ctime) ) @@ -312,10 +214,9 @@ -class MSParser(Parser): - """`Parser` class for MS-specific directory format.""" - +class _MSDirectoryParser(_DirectoryParser): + """`_DirectoryParser` class for MS-specific directory format.""" def parse_line(self, line, time_shift=0.0): """ - Return a `StatResult` instance corresponding to the given + Return a `_StatResult` instance corresponding to the given text line from a FTP server which emits "Microsoft format" (see end of file). @@ -355,8 +256,21 @@ st_atime = None # st_mtime - st_mtime = self.parse_ms_time(date, time_, time_shift) + try: + month, day, year = map(int, date.split('-')) + if year >= 70: + year = 1900 + year + else: + year = 2000 + year + hour, minute, am_pm = time_[0:2], time_[3:5], time_[5] + hour, minute = int(hour), int(minute) + except (ValueError, IndexError): + raise ftp_error.ParserError("invalid time string '%s'" % time_) + if am_pm == 'P': + hour = hour + 12 + st_mtime = time.mktime( (year, month, day, hour, + minute, 0, 0, 0, -1) ) # st_ctime st_ctime = None - stat_result = StatResult( + stat_result = _StatResult( (st_mode, st_ino, st_dev, st_nlink, st_uid, st_gid, st_size, st_atime, st_mtime, st_ctime) ) @@ -369,12 +283,11 @@ # Stat'ing operations for files on an FTP server # -class _Stat(object): +class _Stat: """Methods for stat'ing directories, links and regular files.""" - def __init__(self, host): self._host = host self._path = host.path # use the Unix directory parser by default - self._parser = UnixParser() + self._parser = _UnixDirectoryParser() # allow one chance to switch to another parser if the default # doesn't work @@ -400,5 +313,4 @@ # we _can't_ put this check into `FTPHost._dir`; see its docstring path = self._path.abspath(path) - # `listdir` should only be allowed for directories and links to them if not self._path.isdir(path): raise ftp_error.PermanentError( @@ -412,16 +324,20 @@ names = [] for line in lines: - if self._parser.ignores_line(line): - continue - # for `listdir`, we are interested in just the names, - # but we use the `time_shift` parameter to have the - # correct timestamp values in the cache - stat_result = self._parser.parse_line(line, - self._host.time_shift()) - loop_path = self._path.join(path, stat_result._st_name) - self._lstat_cache[loop_path] = stat_result - st_name = stat_result._st_name - if st_name not in (self._host.curdir, self._host.pardir): - names.append(st_name) + try: + # for `listdir`, we are interested in just the names, + # but we use the `time_shift` parameter to have the + # correct timestamp values in the cache + stat_result = self._parser.parse_line(line, + self._host.time_shift()) + loop_path = self._path.join(path, stat_result._st_name) + self._lstat_cache[loop_path] = stat_result + st_name = stat_result._st_name + if st_name not in (self._host.curdir, self._host.pardir): + names.append(st_name) + except ftp_error.ParserError: + # ignore things like "total 17", as found in some + # server listings + if not line.lower().startswith("total"): + raise return names @@ -444,4 +360,6 @@ if path in self._lstat_cache: return self._lstat_cache[path] + # get output from FTP's `DIR` command + lines = [] # Note: (l)stat works by going one directory up and parsing # the output of an FTP `DIR` command. Unfortunately, it is @@ -451,35 +369,28 @@ "can't stat remote root directory") dirname, basename = self._path.split(path) - - # If even the directory doesn't exist and we don't want the - # exception, treat it the same as if the path wasn't found in - # the directory's contents (compare below). The use of `isdir` - # here causes a recursion but that should be ok because that - # will at the latest stop when we've got to the root directory. -# if not self._path.isdir(dirname) and not _exception_for_missing_path: -# return None + lstat_result_for_path = None # loop through all lines of the directory listing; we # probably won't need all lines for the particular path but # we want to collect as many stat results in the cache as # possible - lstat_result_for_path = None lines = self._host_dir(dirname) for line in lines: - if self._parser.ignores_line(line): - continue - stat_result = self._parser.parse_line(line, - self._host.time_shift()) - loop_path = self._path.join(dirname, stat_result._st_name) - self._lstat_cache[loop_path] = stat_result - # needed to work without cache or with disabled cache - if stat_result._st_name == basename: - lstat_result_for_path = stat_result - if lstat_result_for_path is not None: + try: + stat_result = self._parser.parse_line(line, + self._host.time_shift()) + loop_path = self._path.join(dirname, stat_result._st_name) + self._lstat_cache[loop_path] = stat_result + # needed to work without cache or with disabled cache + if stat_result._st_name == basename: + lstat_result_for_path = stat_result + except ftp_error.ParserError: + # ignore things like "total 17", as found in some + # server listings + if not line.lower().startswith("total"): + raise + if lstat_result_for_path: return lstat_result_for_path - # path was not found during the loop + # path was not found if _exception_for_missing_path: - #TODO use FTP DIR command on the file to implicitly use - # the usual status code of the server for missing files - # (450 vs. 550) raise ftp_error.PermanentError( "550 %s: no such file or directory" % path) @@ -521,9 +432,7 @@ # if we stat'ed a link, calculate a normalized path for # the file the link points to - # we don't use `basename` - # pylint: disable-msg=W0612 dirname, basename = self._path.split(path) path = self._path.join(dirname, lstat_result._st_target) - path = self._path.abspath(self._path.normpath(path)) + path = self._path.normpath(path) # check for cyclic structure if path in visited_paths: @@ -550,5 +459,5 @@ # if a `listdir` call didn't find anything, we can't # say anything about the usefulness of the parser - if (method is not self._real_listdir) and result: + if result != []: self._allow_parser_switching = False return result @@ -556,5 +465,5 @@ if self._allow_parser_switching: self._allow_parser_switching = False - self._parser = MSParser() + self._parser = _MSDirectoryParser() return method(*args, **kwargs) else: @@ -562,32 +471,11 @@ def listdir(self, path): - """ - Return a list of items in `path`. - - Raise a `PermanentError` if the path doesn't exist, but - maybe raise other exceptions depending on the state of - the server (e. g. timeout). - """ return self.__call_with_parser_retry(self._real_listdir, path) def lstat(self, path, _exception_for_missing_path=True): - """ - Return a `StatResult` without following links. - - Raise a `PermanentError` if the path doesn't exist, but - maybe raise other exceptions depending on the state of - the server (e. g. timeout). - """ return self.__call_with_parser_retry(self._real_lstat, path, _exception_for_missing_path) def stat(self, path, _exception_for_missing_path=True): - """ - Return a `StatResult` with following links. - - Raise a `PermanentError` if the path doesn't exist, but - maybe raise other exceptions depending on the state of - the server (e. g. timeout). - """ return self.__call_with_parser_retry(self._real_stat, path, _exception_for_missing_path) Index: ftp_stat_cache.py =================================================================== --- ftp_stat_cache.py (revision 813:2d2d01ad2ec5) +++ ftp_stat_cache.py (revision 595:677d7e4f849a) @@ -1,3 +1,3 @@ -# Copyright (C) 2006-2009, Stefan Schwarzer +# Copyright (C) 2006, Stefan Schwarzer # All rights reserved. # @@ -41,7 +41,5 @@ -#TODO move this to `ftp_error.py`! class CacheMissError(Exception): - """Raised if a path isn't found in the cache.""" pass @@ -78,6 +76,4 @@ def enable(self): """Enable storage of stat results.""" - # `enable` is called by `__init__`, so it's not set outside `__init__` - # pylint: disable-msg=W0201 self._enabled = True @@ -129,14 +125,9 @@ raise an exception. """ - #XXX to be 100 % sure, this should be `host.sep`, but I don't - # want to introduce a reference to the `FTPHost` object for - # only that purpose + #XXX to be 100 % sure, this should be `host.sep` assert path.startswith("/"), "%s must be an absolute path" % path try: del self._cache[path] - # don't complain about lazy except clause - # pylint: disable-msg=W0704 except lrucache.CacheKeyError: - # ignore errors pass @@ -177,5 +168,4 @@ # implicitly do an age test which may raise `CacheMissError`; # deliberately ignore the return value `stat_result` - # pylint: disable-msg=W0612 stat_result = self[path] return True Index: tp_sync.py =================================================================== --- ftp_sync.py (revision 818:6c2f19cdd2c6) +++ (revision ) @@ -1,169 +1,0 @@ -# Copyright (C) 2007, Stefan Schwarzer -# All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are -# met: -# -# - Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# -# - Redistributions in binary form must reproduce the above copyright -# notice, this list of conditions and the following disclaimer in the -# documentation and/or other materials provided with the distribution. -# -# - Neither the name of the above author nor the names of the -# contributors to the software may be used to endorse or promote -# products derived from this software without specific prior written -# permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -# ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR -# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -# LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -# NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -# $Id$ - -""" -Tools for syncing combinations of local and remote directories. - -*** WARNING: This is an unfinished in-development version! -""" - -# Sync combinations: -# - remote -> local (download) -# - local -> remote (upload) -# - remote -> remote -# - local -> local (perhaps implicitly possible due to design, but not targeted) - -import os -import shutil - -from ftputil import FTPHost -import ftp_error - -__all__ = ['FTPHost', 'LocalHost', 'Syncer'] - - -# used for copying file objects; value is 64 KB -CHUNK_SIZE = 64*1024 - - -class LocalHost(object): - def open(self, path, mode): - """ - Return a Python file object for file name `path`, opened in - mode `mode`. - """ - # this is the built-in `open` function, not `os.open`! - return open(path, mode) - - def time_shift(self): - """ - Return the time shift value (see methods `set_time_shift` - and `time_shift` in class `FTPHost` for a definition). By - definition, the value is zero for local file systems. - """ - return 0.0 - - def __getattr__(self, attr): - return getattr(os, attr) - - -class Syncer(object): - def __init__(self, source, target): - """ - Init the `FTPSyncer` instance. - - Each of `source` and `target` is either an `FTPHost` or a - `LocalHost` object. The source and target directories, resp. - have to be set with the `chdir` command before passing them - in. The semantics is so that the items under the source - directory will show up under the target directory after the - synchronization (unless there's an error). - """ - self._source = source - self._target = target - - def _mkdir(self, target_dir): - """ - Try to create the target directory `target_dir`. If it already - exists, don't do anything. If the directory is present but - it's actually a file, raise a `SyncError`. - """ - #TODO handle setting of target mtime according to source mtime - # (beware of rootdir anomalies; try to handle them as well) - #print "Making", target_dir - if self._target.path.isfile(target_dir): - raise ftp_error.SyncError("target dir '%s' is actually a file" % - target_dir) - if not self._target.path.isdir(target_dir): - self._target.mkdir(target_dir) - - def _sync_file(self, source_file, target_file): - #XXX this duplicates code from `FTPHost._copyfileobj`; maybe - # implement the upload and download methods in terms of - # `_sync_file`, or maybe not? - #TODO handle `IOError`s - #TODO handle conditional copy - #TODO handle setting of target mtime according to source mtime - # (beware of rootdir anomalies; try to handle them as well) - #print "Syncing", source_file, "->", target_file - source = self._source.open(source_file, "rb") - try: - target = self._target.open(target_file, "wb") - try: - shutil.copyfileobj(source, target, length=CHUNK_SIZE) - finally: - target.close() - finally: - source.close() - - def _sync_tree(self, source_dir, target_dir): - """ - Synchronize the source and the target directory tree by - updating the target to match the source as far as possible. - - Current limitations: - - _don't_ delete items which are on the target path but not on the - source path - - files are always copied, the modification timestamps are not - compared - - all files are copied in binary mode, never in ASCII/text mode - - incomplete error handling - """ - self._mkdir(target_dir) - for dirpath, dirnames, filenames in self._source.walk(source_dir): - for dirname in dirnames: - inner_source_dir = self._source.path.join(dirpath, dirname) - inner_target_dir = inner_source_dir.replace(source_dir, - target_dir, 1) - self._mkdir(inner_target_dir) - for filename in filenames: - source_file = self._source.path.join(dirpath, filename) - target_file = source_file.replace(source_dir, target_dir, 1) - self._sync_file(source_file, target_file) - - def sync(self, source_path, target_path): - """ - Synchronize `source_path` and `target_path` (both are strings, - each denoting a directory or file path), i. e. update the - target path so that it's a copy of the source path. - - This method handles both directory trees and single files. - """ - #TODO handle making of missing intermediate directories - source_path = self._source.path.abspath(source_path) - target_path = self._target.path.abspath(target_path) - if self._source.path.isfile(source_path): - self._sync_file(source_path, target_path) - else: - self._sync_tree(source_path, target_path) - Index: ftpsync-0.1/LICENSE =================================================================== --- ftpsync-0.1/LICENSE (revision 523:1b5db4835f74) +++ ftpsync-0.1/LICENSE (revision 523:1b5db4835f74) @@ -0,0 +1,30 @@ +# Copyright (C) 2006 Martin Wilck +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# - Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# +# - Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# +# - Neither the name of the above author nor the names of the +# contributors to the software may be used to endorse or promote +# products derived from this software without specific prior written +# permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR +# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +# LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +# NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. Index: ftpsync-0.1/README =================================================================== --- ftpsync-0.1/README (revision 523:1b5db4835f74) +++ ftpsync-0.1/README (revision 523:1b5db4835f74) @@ -0,0 +1,47 @@ +Copyright (c) Martin Wilck 2006 + +See the file LICENSE for copyright information. + +ftpsync is a tool for mirroring (uploading) data to FTP servers written in Python. +It is built upon functionality in Stefan Schwarzer's ftputil package. +It has been tested with Python 2.3 and 2.4 and ftputil 2.1 under Linux. + +Usage: ftpsync.py [options] host source-dir target-dir +Known options: --exclude=, + --include=, + --exclude-from=, + --include-from=, + --delete, + --delete-excluded, + --dry-run, + --verbose, --quiet, --debug, + --trace=, + --cache-expire=, + --cache-size= + +Most options are equivalent to rsync(1)'s respective options. + +Features: + * include/exclude logic like rsync(1). + (Note: there is a script called rsync.py in the Python package index. + I have tested it and found it did not mimic rsync's logic correctly). + * Caching of FTP directory contents (simple FTPHost._dir() caching, + but speed up can be quite big) + * Deals with case-insensitive FTP server + +TODO: * download script + * proper packaging + * ... + +Files in this directory: + + ftpsync.py: main script + + caching_ftp.py: CachingFTPHost object, derived from ftputil + casepath.py: case-insensitive 'path' and 'stat' objects, derived from ftputil + + sync.py: abstract synchronizing logic + rsyncmatch.py: rsync-style globbing and include/exclude patterns + casestr.py: case-insensitive string class + simplecache.py: a very simplistic cache implementation + loggingclass.py: a small commodity layer above 'logging' Index: ftpsync-0.1/caching_ftp.py =================================================================== --- ftpsync-0.1/caching_ftp.py (revision 523:1b5db4835f74) +++ ftpsync-0.1/caching_ftp.py (revision 523:1b5db4835f74) @@ -0,0 +1,127 @@ +from ftputil import FTPHost +from ftputil.ftp_error import PermanentError, InternalError +from loggingclass import LoggingClass +from simplecache import Cache +from casepath import CaseInsPath, CaseInsStat + +class CachingFTPHost(FTPHost, LoggingClass): + + """ + This class is like ftputil.FTPHost, except that + the working directory and directory contents are cached. + This may speed up FTP operations significantly, especially + when traversing trees and looking for stat() like information. + + However, cached information may be wrong in some cases. + It is recommended to call host.invalidate_dir() after + closing the ftp_file object associated with . + + Constructor keywords "expire" and "size" are the same as for + simplecache.Cache. + + Besides, this class adds a method check_case_insensitive() to cope + with FTP servers that don't distinguish file names by case. + + """ + + def __init__(self, *args, **kwargs): + + kw = {} + if "expire" in kwargs: + kw["expire"] = kwargs["expire"] + del kwargs["expire"] + if "size" in kwargs: + kw["size"] = kwargs["size"] + del kwargs["size"] + self.cache = Cache(**kw) + + FTPHost.__init__(self, *args, **kwargs) + self.CWD = None + self.setcwd() + + def check_case_insensitive(self): + """ + Check for server case-insensivity and + This function must be called with an established connection and + with write permissions (like synchronize_times). + + """ + helper_name = "__CachingFtpHost_Helper__" + try: + self.mkdir(helper_name) + self.lstat(helper_name) # exception if mkdir failed + try: + # if the server is case-insensitive, this will fail + file = self.mkdir(helper_name.lower()) + except PermanentError: + self.logger.warning("Server is case-insensitive") + self.path = CaseInsPath(self) + self._stat = CaseInsStat(self) + self.cache.invalidate_all() + else: + self.logger.info("Server is case-sensitive") + finally: + self.rmdir(helper_name) + + def getcwd(self): + """ + Return cached working directory. + """ + return self.CWD + + def setcwd(self): + """ + Update cached working directory. + """ + self.CWD = self.path.normpath(FTPHost.getcwd(self)) + self.logger.info("New cwd: %s" % self.CWD) + + def chdir(self, path): + FTPHost.chdir(self, path) + self.setcwd() + + def _dir(self, path): + + path = self.path.normcase(path) + try: + lines = self.cache[path] + except KeyError: + self.logger.debug("cache miss: %s" % path) + lines = FTPHost._dir(self, path) + self.cache[path] = lines + else: + self.logger.debug("cache hit: %s" % path) + return lines + + def _invalidate_dir(self, path): + self.logger.debug("invalidating cache for %s" % path) + self.cache.invalidate( + self.path.normcase( + self.path.dirname(self.path.abspath(path)))) + + def file(self, path, mode='r'): + path = self.path.abspath(path) + ret = FTPHost.file(self, path, mode) + if 'w' in mode: + self._invalidate_dir(path) + return ret + + def mkdir(self, path, mode=None): + path = self.path.abspath(path) + FTPHost.mkdir(self, path, mode) + self._invalidate_dir(path) + + def rmdir(self, path): + FTPHost.rmdir(self, path) + self.cache.invalidate(self.path.normcase( + self.path.abspath(path))) + self._invalidate_dir(path) + + def remove(self, path): + FTPHost.remove(self, path) + self._invalidate_dir(path) + + def rename(self, source, target): + FTPHost.rename(self, source, target) + self._invalidate_dir(source) + self._invalidate_dir(target) Index: ftpsync-0.1/casepath.py =================================================================== --- ftpsync-0.1/casepath.py (revision 523:1b5db4835f74) +++ ftpsync-0.1/casepath.py (revision 523:1b5db4835f74) @@ -0,0 +1,113 @@ +import posixpath +from ftputil import ftp_path, ftp_stat +from casestr import CaseInsStr + +class BasePath(ftp_path._Path): + """ + A reimplementation of ftputil.ftp_path._Path which is better suited + as base class than the original _Path. + """ + # ftputil.ftp_path._Path can't be inherited well because of the + # direct posixpath assignments in ftputil.ftp_path._Path.__init__(), + # which subclasses can't overload. + + # Here, dirname(), basename(), etc. are defined as methods which + # can be overloaded in derived classes. + + # This class inherits _Path in order to avoid duplicating + # code. Note that _Path.__init__() isn't called. + + def __init__(self, host, pp=posixpath): + self.pp = pp + self._host = host + + def dirname(self, *args): + return self.pp.dirname(*args) + + def basename(self, *args): + return self.pp.basename(*args) + + def isabs(self, *args): + return self.pp.isabs(*args) + + def commonprefix(self, *args): + return self.pp.commonprefix(*args) + + def join(self, *args): + return self.pp.join(*args) + + def split(self, *args): + return self.pp.split(*args) + + def splitdrive(self, *args): + return self.pp.splitdrive(*args) + + def splitext(self, *args): + return self.pp.splitext(*args) + + def normcase(self, *args): + return self.pp.normcase(*args) + + def normpath(self, *args): + return self.pp.normpath(*args) + + +class CaseInsPath(BasePath): + """ + A "path" implementation that treats paths in a case-insensitive manner. + The only difference to BasePath (and _Path) is that all + returned strings are 'CaseInsStr' objects. + """ + + def dirname(self, path): + return CaseInsStr(BasePath.dirname(self, path)) + + def basename(self, path): + return CaseInsStr(BasePath.basename(self, path)) + + def abspath(self, path): + return CaseInsStr(BasePath.abspath(self, path)) + + def normpath(self, path): + return CaseInsStr(BasePath.normpath(self, path)) + + def normcase(self, path): + return CaseInsStr(path.lower()) + + def join(self, *args): + return CaseInsStr(BasePath.join(self, *args)) + + def split(self, *args): + return [CaseInsStr(x) + for x in BasePath.split(self, *args)] + + def splitext(self, *args): + return [CaseInsStr(x) + for x in BasePath.splitext(self, *args)] + + def splitdrive(self, *args): + return [CaseInsStr(x) + for x in BasePath.splitdrive(self, *args)] + + +class CaseInsStat(ftp_stat._Stat): + """ + A class derived from _Stat that treats file names in a case insensitive + manner. + + E.g. "Spam" will be found and stat'd in a directory listing "spam, eggs". + """ + + def _stat_candidates(self, lines, wanted_name): + """Return candidate lines for further analysis.""" + ret = [line + for line in lines + if CaseInsStr(line).find(wanted_name) != -1] + return ret + + def _real_lstat(self, path, _exception_for_missing_path=True): + + path = CaseInsStr(path) + ret = ftp_stat._Stat._real_lstat(self, path, + _exception_for_missing_path) + return ret Index: ftpsync-0.1/casestr.py =================================================================== --- ftpsync-0.1/casestr.py (revision 523:1b5db4835f74) +++ ftpsync-0.1/casestr.py (revision 523:1b5db4835f74) @@ -0,0 +1,292 @@ +import sys +from types import IntType, StringType + +class CaseInsStr(str): + """ + A reimplementation of the standard string class 'str' which + behaves in a case insensitive manner. + + See the python library documentation ("String methods") for + documentation of the methods. + + The case-insensitivity is greedy, i.e. operations between 'str' + and 'CaseInsStr' objects return 'CaseInsStr' objects. + All methods inherited from 'str' which would normally return + 'str' objects return 'CaseInsStr', including lower() and upper(). + + Use the str() method to convert to a normal, case-sensitive string. + + The string itself is not converted to lower or upper case. +""" + + def cast(self, str): + """ + Cast a "str" object into a "CaseInsStr" object. + """ + return CaseInsStr(str) + + def str(self): + """ + Convert to case-sensitive 'str' object. + """ + return self.__str__() + + def _lower(self): + return str.lower(self) + + def __cmp__(self, other): + """ + "CaseInsStr" objects can be compared with each other or with strings. + Objects are compared case-insensitively. + +>>> Spam = CaseInsStr("Spam") +>>> print "egg" < Spam, Spam >= "egg", "egg" < Spam.str(), Spam == "spam" +True True False True +>>> phrase = [CaseInsStr("Eric"), "the", "half", "a", "Bee"] +>>> phrase.sort() +>>> print phrase +['Bee', 'a', 'Eric', 'half', 'the'] + """ + ret = cmp(self._lower() , other.lower()) + return ret + + def __ne__(self, other): + return self.__cmp__(other) != 0 + + def __eq__(self, other): + return self.__cmp__(other) == 0 + + def __lt__(self, other): + return self.__cmp__(other) == -1 + + def __le__(self, other): + return self.__cmp__(other) != 1 + + def __gt__(self, other): + return self.__cmp__(other) == 1 + + def __ge__(self, other): + return self.__cmp__(other) != -1 + + def __getitem__(self, n): + return CaseInsStr(str.__getitem__(self, n)) + + def __getslice__(self, i, j): + return CaseInsStr(str.__getslice__(self, i, j)) + + def __add__(self, other): + """ +>>> # concatenation +>>> print "The "+CaseInsStr("Lovely ")+"Spam" == "the LOVELY spam" +True +""" + return CaseInsStr(str.__add__(self, other)) + + def __radd__(self, other): + return CaseInsStr(str.__add__(other, self)) + + def __mul__(self, n): + """ +>>> print 4 * CaseInsStr("Spam!") + CaseInsStr("Egg!") * 3 +Spam!Spam!Spam!Spam!Egg!Egg!Egg! +""" + return CaseInsStr(str.__mul__(self, n)) + + def __rmul__(self, n): + return CaseInsStr(str.__rmul__(self, n)) + + def center(self, width): + return CaseInsStr(str.center(self, width)) + + def count(self, sub, *args, **kwargs): + """ +>>> print CaseInsStr(4*"SPAM!").count("spam") +4 +""" + return self._lower().count(sub.lower(), *args, **kwargs) + + def find(self, other): + """ +>>> Love = CaseInsStr("The Lovely Spam") +>>> print Love.find("spam"), Love.rfind("love"), Love.index("ELY") +11 4 7 +""" + return self._lower().find(other.lower()) + + def index(self, other): + return self._lower().index(other.lower()) + + def join(self, seq): + """ +>>> print CaseInsStr("!").join(["spam", "Spam", "SPAM"]) +spam!Spam!SPAM +>>> print CaseInsStr("!").join(["spam", "Spam", "SPAM"]).find("SPAM") +0 +""" + return CaseInsStr(str.join(self, seq)) + + def ljust(self, width): + return CaseInsStr(str.ljust(self, width)) + + def replace(self, old, new, count=None): + """ +>>> # replace +>>> print CaseInsStr(4*"EGG!").replace("egg", "Spam", 3) +Spam!Spam!Spam!EGG! +""" + if count is not None and (type(count) != IntType or count < 0): + raise ValueError, count + old = old.lower() + lwr = self._lower() + n = 0 + idx = 0 + ret = "" + + while True: + i = lwr[idx:].find(old) + if i == -1 or (count is not None and n >= count): + break + ret = ret + str.__getslice__(self, idx, idx+i) + new + n = n + 1 + idx = idx + i + len(old) + + ret = ret + str.__getslice__(self, idx, sys.maxint) + return CaseInsStr(ret) + + def startswith(self, other): + return self._lower().startswith(other.lower()) + + def endswith(self, other): + return self._lower().endswith(other.lower()) + + def rfind(self, other): + return self._lower().rfind(other.lower()) + + def rindex(self, other): + return self._lower().rindex(other.lower()) + + def rjust(self, width): + return CaseInsStr(str.rjust(self, width)) + + def split(self, sep=None, maxsplit=0): + """ +>>> print CaseInsStr("Fiddle de dum, fiddle de dee").split("DE", 2) +['Fiddle ', ' dum, fiddle ', ' dee'] +""" + if sep is None: + return self.__str__().split(sep, maxsplit) + + if type(maxsplit) != IntType: + raise TypeError, maxsplit + if maxsplit < 0: + raise ValueError, maxsplit + + ret = [] + last = 0 + while True: + i = self[last:].find(sep) + if i == -1 or (maxsplit > 0 and len(ret) == maxsplit): + break + ret = ret + [self[last:last+i]] + last = last + i + len(sep) + + ret = ret + [self[last:]] + return ret + + def rsplit(self, sep=None, maxsplit=0): + """ +>>> print CaseInsStr("spam and eggs AND bees And knights").rsplit("and", 2) +['spam and eggs ', ' bees ', ' knights'] +""" + if sep is None: + return self.__str__().rsplit(sep, maxsplit) + + if type(maxsplit) != IntType: + raise TypeError, maxsplit + if maxsplit < 0: + raise ValueError, maxsplit + + ret = [] + last = sys.maxint + while True: + i = self[:last].rfind(sep) + if i == -1 or (maxsplit > 0 and len(ret) == maxsplit): + break + ret = [self[i+len(sep):last]] + ret + last = i + + ret = [self[:last]] + ret + return ret + + def splitlines(self, keepends=None): + return [CaseInsStr(x) + for x in str.splitlines(self, keepends)] + + def _stripchars(self, ch): + if type(ch) is not StringType: + raise TypeError, ch + s = "" + for x in ch: + if x.isalpha(): + u = x.upper() + l = x.lower() + if l != u: + s = s + l + u + continue + s = s + x + return s + + def strip(self, *chars): + """ + Stripped characters are case-insensitive: +>>> print CaseInsStr(" Spam and eggs ").strip("s ") +pam and egg +>>> print "'%s'" % CaseInsStr(" a ").lstrip() +'a ' +""" + if chars is () or chars[0] is None: + return CaseInsStr(str.strip(self)) + else: + return CaseInsStr(str.strip(self, self._stripchars(chars[0]))) + + def lstrip(self, *chars): + if chars is () or chars[0] is None: + return CaseInsStr(str.lstrip(self)) + else: + return CaseInsStr(str.lstrip(self, self._stripchars(chars[0]))) + + def rstrip(self, *chars): + if chars is () or chars[0] is None: + return CaseInsStr(str.rstrip(self)) + else: + return CaseInsStr(str.rstrip(self, self._stripchars(chars[0]))) + + def swapcase(self): + return CaseInsStr(str.swapcase(self)) + + def title(self): + return CaseInsStr(str.title(self)) + + def translate(self): + raise NotImplementedError, "translate() method not implemented" + + def lower(self): + """ +>>> Spam = CaseInsStr("Spam") +>>> print Spam.lower(), Spam.upper(), Spam.lower() == "SPAM", Spam.upper() == "spam" +spam SPAM True True +""" + return CaseInsStr(self._lower()) + + def upper(self): + return CaseInsStr(self.__str__().upper()) + + def zfill(self, width): + return CaseInsStr(str.zfill(self, width)) + +def _test(): + import doctest, casestr + doctest.testmod(casestr) + +if __name__ == "__main__": + _test() Index: ftpsync-0.1/ftpsync.py =================================================================== --- ftpsync-0.1/ftpsync.py (revision 523:1b5db4835f74) +++ ftpsync-0.1/ftpsync.py (revision 523:1b5db4835f74) @@ -0,0 +1,235 @@ +import getopt +import netrc # for password retrieval from .netrc +import os +import sys +import termios # for getpass +import time +import traceback + +import loggingclass + +from caching_ftp import CachingFTPHost +from rsyncmatch import GlobChain +from sync import Synchronizer, RsyncSynchronizer + +# Copied from Python library manual (termios) +def getpass(prompt = "Password: "): + fd = sys.stdin.fileno() + old = termios.tcgetattr(fd) + new = termios.tcgetattr(fd) + new[3] = new[3] & ~termios.ECHO # lflags + try: + termios.tcsetattr(fd, termios.TCSADRAIN, new) + passwd = raw_input(prompt) + finally: + termios.tcsetattr(fd, termios.TCSADRAIN, old) + return passwd + + +def login_data(host): + """ + Derive login data for different FTP host formats: + + "hostname": check .netrc file, try anonymous otherwise + "user:pass@hostname": parse + "user@hostname": parse and ask for password interactively + """ + + user = "" + acct = "" + passwd = "" + + at = host.find("@") + if (at == -1): + try: + nrc = netrc.netrc() + (user, acct, passwd) = \ + nrc.authenticators(host) + except (IOError, TypeError): # no netrc file or no entry in netrc + pass + else: + user = host[:at] + host = host[(at+1):] + col = user.find(":") + if (col == -1): + passwd = getpass("password for ftp://%s%s: " % (user, host)) + else: + passwd = user[(col+1):] + user = user[:col] + + if (user == ""): + user = "anonymous" + + return (host, user, passwd, acct) + +def init_ftp(host, dir, **kw): + """ + Set up an FTP session for synchronizing (upload). + We must have write permissions in this case. + """ + data = login_data(host) + ftp = CachingFTPHost(*data, **kw) + ftp.chdir(dir) + ftp.synchronize_times() + ftp.check_case_insensitive() + return ftp + + +def start_logging(level, logfile): + """ + Initialize a useful logging environment with logging to stderr, + slightly more verbose to a log file, and suitable log levels. + """ + loggingclass.init_logging(level) + + if level == loggingclass.DEBUG: +# loggingclass.getLogger().setLevel(loggingclass.DEBUG) + loggingclass.set_default_level(loggingclass.INFO) + if logfile != "": + loggingclass.init_logfile(logfile, level=loggingclass.DEBUG) + loggingclass.set_class_level(RsyncSynchronizer, loggingclass.DEBUG) + loggingclass.set_class_level(GlobChain, loggingclass.DEBUG) + loggingclass.set_class_level(CachingFTPHost, loggingclass.INFO) + else: + if logfile != "": + loggingclass.init_logfile(logfile, level=loggingclass.INFO) + loggingclass.set_class_level(RsyncSynchronizer, loggingclass.INFO) + loggingclass.set_class_level(CachingFTPHost, loggingclass.INFO) + +def print_err(): + err = sys.exc_info() + printit = True + try: + if parm.level != loggingclass.DEBUG: + printit = False + except: + pass + if printit: + traceback.print_tb(err[2]) + sys.stderr.write("%s: %s\n" % (err[0], err[1])) + +class _Params: + """ + Class representing options and arguments for do_sync(). + """ + + class UsageError(Exception): + pass + + known = (list(GlobChain().options()) + + ["delete", "delete-excluded", "dry-run", + "verbose", "quiet", "debug", "trace=", + "cache-expire=", "cache-size="]) + + def usage(self): + sys.stderr.write("""\ +Usage: %s [options] host source-dir target-dir +Known options: %s +""" % (sys.argv[0], ", ".join(["--" + x for x in self.known]))) + raise self.UsageError() + + def _setlevel(self, level, option=True): + if self.level == -1: + self.level = level + elif option: + sys.stderr.write("Only one of --quiet, --debug, --verbose may be specified\n") + self.usage() + + def __init__(self): + self.level = -1 + self.dry_run = False + self.delete = False + self.delete_excluded = False + self.logfile = "" + self.expire = 300 + self.size = 2000 + + try: + (opts, args) = getopt.gnu_getopt(sys.argv[1:], "", self.known) + except getopt.GetoptError: + print_err() + self.usage() + + for (o, v) in opts: + if o == "--delete": + self.delete = True + elif o == "--delete-excluded": + self.delete_excluded = True + elif o == "--dry-run": + self.dry_run = True + elif o == "--verbose": + self._setlevel(loggingclass.INFO) + elif o == "--debug": + self._setlevel(loggingclass.DEBUG) + elif o == "--quiet": + self._setlevel(loggingclass.WARNING) + elif o == "--trace": + self.logfile=v + elif o == "--cache-expire": + self.expire = int(v) + elif o == "--cache-size": + self.size = int(v) + + self._setlevel(loggingclass.NOTICE, option=False) + if len(args) != 3: + self.usage() + + (self.host, self.source, self.target) = args + self.opts = opts + + +def _fix_source_n_target(parm): + # rsync-like semantics for trailing slash in source dir + if not os.path.isdir(parm.source): + raise ValueError, "%s is not a directrory" % parm.source + + if parm.source.endswith(os.sep): + parm.source = parm.source.rstrip(os.sep) + else: + parm.target = parm.ftp.path.join(parm.target, + os.path.basename(parm.source)) + if not parm.ftp.path.exists(parm.target): + parm.ftp.mkdir(parm.target) + +def log_checkpoint(msg): + loggingclass.getLogger().info( + "%s %s %s" % (__file__, msg, + time.strftime("%Y-%m-%d, %H:%M", time.localtime()))) + +def do_sync(parm): + + start_logging(parm.level, parm.logfile) + log_checkpoint("starting at") + + if parm.host == "localhost": + parm.ftp = os + else: + parm.ftp = init_ftp(parm.host, parm.target, + expire=parm.expire, size=parm.size) + + _fix_source_n_target(parm) + + sync = RsyncSynchronizer(os, parm.ftp, parm.source, parm.target, + delete = parm.delete, + dry_run = parm.dry_run, + delete_excluded = parm.delete_excluded) + + sync.globchain.getopt(parm.opts) + sync.sync("") + + log_checkpoint("finished at") + +if __name__ == "__main__": + try: + parm = _Params() + do_sync(parm) + except KeyboardInterrupt: + sys.stderr.write("Interrupted.\n") + sys.exit(0) + except _Params.UsageError: + sys.exit(129) + except: + print_err() + sys.exit(130) + else: + sys.exit(0) Index: ftpsync-0.1/loggingclass.py =================================================================== --- ftpsync-0.1/loggingclass.py (revision 523:1b5db4835f74) +++ ftpsync-0.1/loggingclass.py (revision 523:1b5db4835f74) @@ -0,0 +1,118 @@ +from logging import * +import sys + +NOTICE = (INFO+WARNING)/2 +_default_level = NOTICE +_default_format = "%(filename)s:%(name)s[%(lineno)d]: %(message)s" + +def set_default_level(level): + """ + Set the default log level for classes for which set_class_level() + or instance.set_log_level() was never called. + Default: NOTICE. + Call early - only affects class loggers created after the call. + """ + global _default_level + _default_level = level + +def _class_logger(cls): + nm ="_" + cls.__name__ + "__logger" + try: + lg = getattr(cls, nm) + except AttributeError: + lg = getLogger(cls.__name__) + lg.setLevel(_default_level) + setattr(cls, nm, lg) + return lg + +def set_class_level(cls, level): + """ + Set the log level for this class. + level: one of the levels defined in the logging module. + Side effects: Sets the log level for _any object_ of this class. + """ + lg = _class_logger(cls) + lg.setLevel(level) + +class LoggingClass: + + """ + A base class for classes using for easy use of the "logging" module. + + Instances of derived classes have a "logger" attribute + that represents a class-specific logging.Logger object. + + The "name" of the class-specific logger will be the class name. + Log levels etc. can be set on a class-specific basis. + + CAUTION: The "logger" attribute is resolved through __getattr__(). + The "__logger" attribute should work independently of __getattr__(). + + Usage (doctest): + +>>> class LogTest(LoggingClass): +... def info(self): +... self.logger.info("This is an info.") +... def error(self): +... self.logger.error("This is an error!") +... +>>> init_logging(level=INFO, +... format="%(name)s[%(lineno)d]: %(message)s", stream=sys.stdout) +>>> logtest = LogTest() +>>> logtest.info() +>>> logtest.error() +LogTest[5]: This is an error! +>>> LogTest().set_log_level(INFO) +>>> logtest.info() +LogTest[3]: This is an info. + + """ + + # We can't simply define a "logger" attribute because we + # want class-specific loggers. + + # The attribute name is constructed such that self.__logger + # will work in derived classes. + + def __getattr__(self, attr): + """ + Resolves the "logger" attribute. + Call this when redefining __getattr__() in derived classes! + """ + if attr == "logger": + return _class_logger(self.__class__) + raise AttributeError, ("'LoggingClass' object has no attribute '%s'" + % attr) + + def set_log_level(self, level): + """ + Set the log level for this class. + level: one of the levels defined in the logging module. + Sets the log level for _any object_ of this class. + """ + return set_class_level(self.__class__, level) + + +def init_logging(level=NOTICE, format=_default_format, stream=sys.stderr): + """Initialize a basic logging setup.""" + + handler = StreamHandler(stream) + handler.setFormatter(Formatter(format)) + handler.setLevel(level) + + getLogger().addHandler(handler) + +def init_logfile(file, level=INFO, format=_default_format): + + handler = FileHandler(file) + handler.setFormatter(Formatter(format)) + handler.setLevel(level) + + getLogger().addHandler(handler) + +def _test(): + import doctest, loggingclass + doctest.testmod(loggingclass) + +if __name__ == "__main__": + _test() Index: ftpsync-0.1/rsyncmatch.py =================================================================== --- ftpsync-0.1/rsyncmatch.py (revision 523:1b5db4835f74) +++ ftpsync-0.1/rsyncmatch.py (revision 523:1b5db4835f74) @@ -0,0 +1,437 @@ +import re +import os +import sys +import loggingclass + +INCLUDE = "+" +EXCLUDE = "-" +DONE = "." + +class RsyncGlob(loggingclass.LoggingClass): + """ + A class that imitates rsync(1)'s way of include/exclude patterns. + Similar to glob()/fnmatch(), but "*" doesn't match "/" - "**" does. + See man rsync(1) for the exclude/include logic. + The GlobChain class creates filter chains like rsync's. + + RsyncGlob(pattern), where follows the rules in the rsync(1) + man page. In particular, "/XYZ" matches only at the "root", and "XYZ/" + matches only directories. + + NOTE: This class uses Unix file name conventions. + It will be pretty simple to implement it for DOS/Windows though, + if someone volunteers. + + The following doctest example shows how the globbing works. + Note that leading "/" are stripped for files. + +>>> globs=(RsyncGlob("s*m"),RsyncGlob("s**m"), +... RsyncGlob("s\*m"),RsyncGlob("/s*m"), +... RsyncGlob("/s**m"),RsyncGlob("s\**m")) +>>> files=("spam","s/p/a/m","egg/spam","egg/sp/am","s*m","s*am") +>>> def outh(): +... s="%10.10s" %"" +... for g in globs: +... s=s+"%10.10s"%g.glob +... return s +>>> def outf(f): +... s="%10.10s" %f +... for g in globs: +... s=s+"%10.10s"%g.match(f) +... return s +>>> def out(): +... print outh() +... for f in files: +... print outf(f) +>>> out() + s*m s**m s\*m /s*m /s**m s\**m + spam True True False True True False + s/p/a/m False True False False True False + egg/spam True True False False False False + egg/sp/am False True False False False False + s*m True True True True True True + s*am True True False True True True + """ + + # This is applied before escaping re metachars + __bksl_re = re.compile(r'(\\.)') + + # These are applied after escaping re metachars + __star2_re = re.compile(r'(\\\*\\\*)') # "**" + __star_re = re.compile(r'(\\\*)') # "*" + __quest_re = re.compile(r'(\\\?)') # "?" + __slash_re = re.compile(r'/') # "/" + + def __handle_stars(self, s): + parts = self.__star2_re.split(s) + + # side effect: patterns containing "**" match complete path + self.path_match = self.path_match or (len(parts) > 1) + + # Odd elements are '**' now. + # Even elements must be checked for '*' and '?' + res = "" + i = 0 + while i < len(parts): + if parts[i] != "": + tmp = self.__star_re.sub("[^/]*", parts[i]) + tmp = self.__quest_re.sub("[^/]", tmp) + res = res + tmp + if i < len(parts) - 1: + res = res + ".*" + i = i + 2 + return res + + + def __init__(self, pat=""): + + self.type = None + + # patterns starting with +/-. + if len(pat) > 1: + if pat[:2] == "+ ": + self.type = INCLUDE + pat = pat[2:] + elif pat[:2] == "- ": + self.type = EXCLUDE + pat = pat[2:] + + # patterns ending with "/" match only directories + if len(pat) > 0 and pat.endswith("/"): + self.dir_match = True + pat = pat[:-1] + else: + self.dir_match = False + + self.glob = pat + + # patterns containing "/" match entire path, + self.path_match = (pat.find("/") != -1) + + # patterns starting with "/" match only at root + if len(pat) > 0 and pat[0] == "/": + pat = pat[1:] + top_match = True + else: + top_match = False + + # We transform the glob pattern into a regexp pattern now. + # First, handle all characters escaped with backslashes. + parts = self.__bksl_re.split(pat) + + i = 0 + self.pat = "" + + # Odd elements of parts are an escaped chars now. + # Need to look for glob patterns in even elements. + while i < len(parts): + if parts[i] != "": + # escape any remaining regexp metacharacters like "." + s = re.escape(parts[i]) + # sort out "**" and "*" + s = self.__handle_stars(s) + self.pat = self.pat + s + # Add back the escaped chars + if (i < len(parts) - 1): + self.pat = self.pat + parts[i+1] + i = i+2 + + self.pat = self.pat + "$" + if top_match: + self.pat = "^" + self.pat + # Special case: '**/' matches empty string + elif self.pat[:4] == r".*\/": + self.pat = "(.*/|)" + self.pat[4:] + + self.logger.debug("regexp: %s -> (%s)" % (self, self.pat)) + self.re = re.compile(self.pat) + + + def __str__(self): + s = self.glob + if self.dir_match: s = s + "/" + + if self.type: + t = self.type + else: + t = " " + if self.path_match: + p="p" + else: + p=" " + return "(%s)[%s%s]" % (s, t, p) + + + def match(self, filename): + + if len(filename) > 0 and filename.endswith("/"): + filename = filename [:-1] + elif self.dir_match: + return False + + if self.path_match: + ret = self.re.search(filename) is not None + else: + ret = self.re.match(os.path.basename(filename)) is not None + + if ret: + self.logger.debug("%s matches %s" % (filename, self)) + return ret + + +class GlobChain(loggingclass.LoggingClass): + """ + A class that represents a chain of RsyncGlob filter rules. + Filter rules are applied in order. The recurse() function can + be used to filter directories recursively. + + doctest example: + +>>> loggingclass.init_logging(level=loggingclass.DEBUG, +... format="%(name)s[%(lineno)d]: %(message)s", +... stream=sys.stdout) +>>> +>>> ch = GlobChain() +>>> ch.set_log_level(loggingclass.DEBUG) +>>> +>>> ch.exclude("+ spam/", "- /*/", "+ egg/", "- */", "+ \*", "- *") +GlobChain[251]: added rule: (spam/)[+ ] +GlobChain[251]: added rule: (/*/)[-p] +GlobChain[251]: added rule: (egg/)[+ ] +GlobChain[251]: added rule: (*/)[- ] +GlobChain[251]: added rule: (\*)[+ ] +GlobChain[251]: added rule: (*)[- ] +>>> for x in ("spam", "spam/", "egg", +... "egg/", "*", "spam/egg", +... "spam/egg/", "spam/*/egg", "spam/egg/*"): +... xx = ch.match(x) +GlobChain[279]: exclude spam +GlobChain[279]: include spam/ +GlobChain[279]: exclude egg +GlobChain[279]: exclude egg/ +GlobChain[279]: include * +GlobChain[279]: exclude spam/egg +GlobChain[279]: include spam/egg/ +GlobChain[279]: exclude spam/*/egg +GlobChain[279]: include spam/egg/* + """ + + __end_re = re.compile(r"/+$") + + def __init__(self): + + self._lst = [] + + def _in_ex(self, x): + if x == INCLUDE: + return "include" + elif x == EXCLUDE: + return "exclude" + else: + return None + + def add(self, type, *args): + """ + add(type, *args): add (a) new INCLUDE/EXCLUDE pattern rule(s). + is either INCLUDE or EXCLUDE, or the patterns must + start with "+" or "-". + +/- start character has precedence over . + """ + for x in args: + # "!" resets the rule chain (why?) + if (x == "!"): + l = [] + else: + glb = RsyncGlob(x) + if (glb.type == None): + if (type == None): + raise ValueError, "filter type is undefined for %s" % glb + else: + glb.type = type + self.logger.info("added rule: %s" % glb) + self._lst.append(glb) + + def exclude(self, *args): + """ + exclude(*args): add (a) new EXCLUDE pattern rule(s) + """ + self.add(EXCLUDE, *args) + + def include(self, *args): + """ + include(*args): add (a) new INCLUDE pattern rule(s) + """ + self.add(INCLUDE, *args) + + def match(self, path): + """match(path): returns the result of the current filter chain for path. + The rule chain is traversed until the first rule matches. + If path is a directory, it should end in "/". + """ + + # Default is always INCLUDE + ret = INCLUDE + for glb in self._lst: + if glb.match(path): + ret = glb.type + break + + self.logger.debug("%s %s" % (self._in_ex(ret), path)) + return ret + + def _recurse(self, top, dir, collector, *args): + + for f in os.listdir(os.path.join(top, dir)): + rel = os.path.join(dir, f) + isdir = os.path.isdir(os.path.join(top, rel)) + + pat = rel + if (isdir): + pat = pat + "/" + + c = self.match(pat) + + collector(c, pat, *args) + if c == EXCLUDE: + continue + + if isdir: + self._recurse(top, rel, collector, *args) + + def collect(self, c, x, *args): + """ + Default collector function to use with recurse(). + """ + l = args[0] + if c == INCLUDE: + l.append(x) + elif c == DONE: + return l + + def recurse(self, dir, collector = None, *args): + """ + recurse(self, dir, collector = None, *args) + recursively descend . For each file or directory found, + is called with arguments (c, x, *args), where + is the result of the test chain (or DONE when finished), + is the current path, and <*args> is the rest of the arguments + of recurse(). + + Directories for which the chain evaluates to EXCLUDE are never entered. + + When c == DONE, should return its final result. This will + be the return value of recurse(). + + If isn't set, a default collector function is used that + returns a list of all files below for which c == INCLUDE. + """ + + if collector == None: + collector = self.collect + args = ([],) + + dir = self.__end_re.sub("", dir) + if not os.path.isdir(dir): + raise IOError, "%s is not a directory" % dir + + self._recurse(dir, "", collector, *args) + ret = collector(DONE, None, *args) + return ret + + def add_file(self, type, name): + """ + add_file(type, name): + Add a list of INCLUDE/EXCLUDE filter rules from a file. + Used to implement --exclude-from, --include-from. + """ + try: + if name == "-": + f = sys.stdin + else: + f = open(name, "r") + + for line in f.readlines(): + if line.endswith("\n"): + line = line[:-1] + self.add(type, line) + finally: + if name != "-": + f.close() + + _known_options = ("exclude=", "include=", + "exclude-from=", "include-from=") + + def options(self): + return self._known_options + + def getopt(self, options): + """ + getopt(options): parse getopt-style option pairs for filter rules. + Parses all options "--exclude=", "--include=", "--exclude-from=", + "--include-from=", leaving other options untouched. + See rsync(1) for the option semantics. + """ + hits = [] + for i in range(0, len(options)): + (name, val) = options[i] + hit = True + if name == "--exclude": + self.exclude(val) + elif name == "--include": + self.include(val) + elif name == "--exclude-from": + self.add_file(EXCLUDE, val) + elif name == "--include-from": + self.add_file(INCLUDE, val) + else: + hit = False + if hit: + hits.append(i) + + hits.reverse() + for i in hits: + del options[i] + + +def print_matches(): + """ + Usage: rsyncmatch.py [--debug] [filter rules...] directory ... + + Print list of files below that match the given rules. + Rules are specified with "--exclude=", "--include=", "--exclude-from=", + "--include-from=", see rsync(1) for rule semantics. + """ + + import getopt + import logging + + loggingclass.init_logging() + + + gl = GlobChain() + (options, args) = getopt.gnu_getopt(sys.argv[1:], "", + (gl._known_options) + ("debug",)) + gl.getopt(options) + for x in options: + if x[0] == "--debug": + GlobChain().set_log_level(loggingclass.DEBUG) + RsyncGlob().set_log_level(loggingclass.DEBUG) + + for dir in args: + ls = gl.recurse(dir) + + print "List of include files below %s:" % dir + for x in ls: + print " " + x + + +def _test(): + import doctest, rsyncmatch + doctest.testmod(rsyncmatch) + +if __name__ == "__main__": + if sys.argv[1] == "--test": + sys.argv = sys.argv[1:] + _test() + else: + print_matches() Index: ftpsync-0.1/simplecache.py =================================================================== --- ftpsync-0.1/simplecache.py (revision 523:1b5db4835f74) +++ ftpsync-0.1/simplecache.py (revision 523:1b5db4835f74) @@ -0,0 +1,225 @@ +import threading +import loggingclass +import time + +class CacheEntry: + """ + A class representing a cache entry. + x = CacheEntry() + """ + + def __init__(self, val): + self.val = val + self.stamp = time.time() + + def expired(self, period): + """ + Boolean: returns true if the entry is older than period (in sec). + """ + return time.time() - self.stamp > period + + def __cmp__(self, other): + """ + CacheEntry objects can be compared by age. + """ + return cmp(self.stamp, other.stamp) + + +class Cache(loggingclass.LoggingClass): + """ + A simple cache implementation + + Usage: c = Cache(expire=, size=, entryclass=CacheEntry) + : expiration time of entries in sec (default: 60) + : max number of cache entries (default: 1000) + : A class for the cache entries (default: CacheEntry) + + NOTE: EXPIRED ENTRIES WILL BE DELETED. + Do not use this class (exclusively) to store valuable data. + + Entries are assigned and retrieved through indexing: + cache[x] = val + try: + val = cache[x] + except KeyError: + print x, ": not in cache" + cache.invalidate(x) + + doctest example: + +>>> from time import sleep +>>> exp=2.0 +>>> cache = Cache(size=10, expire=exp) +>>> for x in range(0, 10): +... cache[x] = x*x +>>> print cache.contents() +[(0, 0), (1, 1), (2, 4), (3, 9), (4, 16), (5, 25), (6, 36), (7, 49), (8, 64), (9, 81)] +>>> for x in range(11, 20): +... cache[x] = x*x +>>> +>>> # size exceeded - old elements will be deleted +>>> print cache.contents() +[(11, 121), (12, 144), (13, 169), (14, 196), (15, 225), (16, 256), (17, 289), (18, 324), (19, 361)] +>>> sleep(exp/2.) +>>> for x in range(0, 5): +... cache[x] = x*x +>>> print cache.contents() +[(0, 0), (1, 1), (2, 4), (3, 9), (4, 16), (16, 256), (17, 289), (18, 324), (19, 361)] +>>> sleep(exp/2.+0.1) +>>> +>>> # elements 11 .. 20 will be expired +>>> print cache.contents() +[(0, 0), (1, 1), (2, 4), (3, 9), (4, 16)] + """ + + default_expire = 60 + default_size = 1000 + _to_shrink = 100 # No. of entries to delete in a _shrink() call + + def __init__(self, expire = default_expire, size = default_size, + entryclass = CacheEntry): + self.__cache = {} + self.__size = size + self.expire = expire + self.__lock = threading.Lock() + self._entryclass = entryclass + + def _lock(self): + self.__lock.acquire() + + def _unlock(self): + self.__lock.release() + + def _invalidate(self, key): + if self.__cache.has_key(key): + del self.__cache[key] + return True + return False + + def len(self): + """ + Return current number of cache entries. + """ + return len(self.__cache) + + def invalidate(self, key): + """ + Invalidate (delete) cache entry indexed by key + """ + self._lock() + try: + if self._invalidate(key): + self.logger.debug("element %s invalidated" % key) + finally: + self._unlock() + + def invalidate_all(self): + """ + Clear cache completetly + """ + self._lock() + try: + self.__cache.clear() + finally: + self._unlock() + self.logger.info("cache cleared") + + def invalidate_some(self, func): + """ + Invalidate all entries for which func(key,val) returns True + """ + self._lock() + try: + for x in self.__cache.keys(): + if func(x, self.__cache[x].val): + self._invalidate(x) + self.logger.debug("element %s invalidated" % x) + finally: + self._unlock() + + def _expired(self, entry): + return self.expire != 0 and entry.expired(self.expire) + + def __getitem__(self, key): + """ + Implements x = cache[key]. + Will raise KeyError if the cache entry is expired. + """ + ret = None + self._lock() + try: + entry = self.__cache[key] + if self._expired(entry): + self._gc() + raise KeyError + else: + ret = entry.val + finally: + self._unlock() + + return ret + + def __setitem__(self, key, val): + + """ + Implements cache[key] = y. + """ + self._lock() + try: + self._invalidate(key) + if self.len() >= self.__size: + self._shrink() + self.__cache[key] = self._entryclass(val) + finally: + self._unlock() + + def _shrink(self): + # Must be called with lock held + target = self.__size - min(self.__size/2, self._to_shrink) + self.logger.debug("_shrink: trying to reduce size from %d to %d" + % (self.len(), target)) + + self._gc() + + n = self.len() - target + if n <= 0: + return + + # sort entries by age and invalidate the n oldest + keys = self.__cache.keys() + keys.sort(lambda a, b, c=self.__cache: cmp(c[a], c[b])) + + self.logger.info("_shrink: invalidating %d entries" % n) + for key in keys[:n]: + self._invalidate(key) + + def _gc(self): + # Must be called with lock held + expired = [] + for key in self.__cache.keys(): + if self._expired(self.__cache[key]): + expired.append(key) + for key in expired: + self._invalidate(key) + self.logger.info("collecting garbage: %d items invalidated" + % len(expired)) + + def contents(self): + """ + returns a list of (key, val) tuples for all non-expired entries + """ + self._lock() + try: + self._gc() + ret = [(x, self.__cache[x].val) for x in self.__cache.keys()] + finally: + self._unlock() + return ret + + +def _test(): + import doctest, simplecache + doctest.testmod(simplecache) + +if __name__ == "__main__": + _test() Index: ftpsync-0.1/sync.py =================================================================== --- ftpsync-0.1/sync.py (revision 523:1b5db4835f74) +++ ftpsync-0.1/sync.py (revision 523:1b5db4835f74) @@ -0,0 +1,388 @@ +import os +import sys +from loggingclass import LoggingClass, NOTICE +from rsyncmatch import GlobChain, EXCLUDE + +class Synchronizer(LoggingClass): + """ + A class for synchronizing directories between two file systems. + + Usage example: + sync = Synchronizer(os, os, "/source", "/target") + sync.sync("subdir") + + This will synchronize "/source/subdir" to "/target/subdir". + """ + + class SyncAction: + """ + This "class" stores actions to be carried out. + """ + def __init__(self): + self.unl = [] # stuff to unlink + self.cpy = [] # stuff to copy + self.rmd = [] # stuff to rmdir + self.mkd = [] # stuff to mkdir + self.dsc = [] # dirs to descend into + + class FileSys: + """ + A helper class for Synchronizer. Another abstraction layer above + 'os' and other filesystem access (e.g. FTP). + + It inherits most attributes from it's '_io' element (typically 'os'). + """ + + def __init__(self, io, root): + self._io = io + self.root = root + + def open(self, *args): + """ + Open a file on the file system. + """ + if self._io == os: + return open(*args) + else: + return self._io.open(*args) + + def eq(self, x, y): + """ + Boolean: True if file names x and y are equal by this file + system's rules. This refers mainly to case-sensitiveness. + """ + ret = (self._io.path.normcase(x) == self._io.path.normcase(y)) + return ret + + def cmp(self, x, y): + """ + Compare file names by this file system's rules. + """ + ret = cmp(self._io.path.normcase(x), self._io.path.normcase(y)) + return ret + + def __getattr__(self, attr): + return getattr(self._io, attr) + + + def __init__(self, io_s, io_t, root_s, root_t, + mode = "b", blocksize = 65536, + delete=False, delete_excluded=False, + dry_run=False): + """ + io_s, io_t: "IO class" of the source and target, respectively. + typically 'os' or an ftputil.FTPHost + root_s, root_t: root directories for synchronization on source + and target, respectively. + mode: file open() mode (usually 'b') + blocksize: block size for copying (default: 64kB) + delete: whether to delete additional files on target (default: false) + delete_excluded: whether to delete files which were excluded, similar + to rsync's --delete-exluded option. See exclude() method. + dry_run: whether anything should actually be done on the target. + """ + + self.io_s = self.FileSys(io_s, root_s) + self.io_t = self.FileSys(io_t, root_t) + self.mode = mode + self.dry_run = dry_run + self.blocksize = blocksize + self.delete = delete + self.delete_excluded = delete_excluded + self.logger.info("options: delete=%s, delete-excluded=%s, dry-run=%s" + % (self.delete, self.delete_excluded, self.dry_run)) + return + + def _rm_rf(self, path): + err = False + for f in self.io_t.listdir(path): + absl = self.io_t.path.join(path, f) + if self.isdir(self.io_t, absl): + try: + self._rm_rf(absl) + except OSError: + self.logger.exception("rmdir %s" % absl) + err = sys.exc_info()[:2] + else: + self.logger.debug("delete %s" % absl) + try: + if not self.dry_run: + self.io_t.unlink(absl) + except OSError: + self.logger.exception("delete %s" % absl) + err = sys.exc_info()[:2] + self.logger.debug("rmdir %s" % path) + if not self.dry_run: + self.io_t.rmdir(path) + if err: + raise err[0], err[1] + + def rm_rf(self, path): + """ + Remove directory recursively. + """ + absl=self.io_t.path.abspath(path) + self._rm_rf(absl) + + def _pull(self, x, lst, eq): + """ + x: file name + lst: list of file names + eq: function to check file name equality + returns: true if file was matched + side effects: removes all matching entries from lst + """ + oldlen = len(lst) + i = 0 + while i < len(lst): + if eq(x, lst[i]): + found = True + del lst[i] + i = i + 1 + return (len(lst) < oldlen) + + def exclude(self, dir, name, isdir): + """ + (virtual): this implementation returns always False. + dir: parent directory + name: file name + isdir: True iff name represents a directory itself + returns: True if file is to be excluded. + """ + return False + + def need_copy(self, src, tgt): + """ + src, tgt: corresponding files on source and target + returns: a "reason string" if src needs to be copied to tgt. + the emtpy string otherwise. + This default implementation returns non-"" if the file + sizes differ ("size"), or if src is newer than tgt ("date"). + """ + ret = "" + stat_s = self.io_s.stat(src) + stat_t = self.io_t.stat(tgt) + if stat_s.st_size != stat_t.st_size: + ret = "size" + self.logger.debug("%s: sizes differ: %d %d" % + (tgt, stat_s.st_size, stat_t.st_size)) + elif (stat_s.st_mtime - stat_t.st_mtime > 0): + ret = "date" + self.logger.debug("%s: source is newer by %s s" % + (tgt, stat_s.st_mtime - stat_t.st_mtime)) + return ret + + def _make_pattern(self, path, isdir): + if isdir: + path = path + "/" + return path + + def isdir(self, io, path): + return io.path.isdir(path) and not io.path.islink(path) + + def copy(self, abs_s, abs_t): + try: + src = self.io_s.open(abs_s, "r" + self.mode) + tgt = self.io_t.open(abs_t, "w" + self.mode) + while True: + buffer = src.read(self.blocksize) + if not buffer: break + tgt.write(buffer) + except(IOError, OSError): + self.logger.exception("error copying to %s" % abs_t) + try: + self.io_s.unlink(abs_t) + except(IOError, OSError): + self.logger.exception("error unlinking %s" % abs_t) + pass + + try: + src.close() + tgt.close() + except: + pass + + def _unique(self, lst, eq): + """ + Remove duplicate entries in list lst, using equality relation eq. + """ + i = 0 + while i < len(lst): + j = i + 1 + while j < len(lst): + if eq(lst[i], lst[j]): + self.logger.warn("skipping %s (duplicate of %s)" + % (lst[j], lst[i])) + del lst[j] + else: + j = j + 1 + i = i + 1 + + def sync(self, path, _top=True): + """ + Main work horse of Synchorinzer class. + Synchronize directory 'path' between source and target. + + Called recursively. Call with _top = True initially. + """ + + # All action items are recorded in this "todo" list. + # Actions are only put into effect when the list is + # complete. + todo = self.SyncAction() + # 'reason' is a map that stores the reasons why we transfer + # files (regular files only). This is just informational. + reason = {} + + path_s = self.io_s.path.join(self.io_s.root, path) + path_t = self.io_t.path.join(self.io_t.root, path) + + if _top: + self.logger.info("sync starting: %s -> %s" % (path_s, path_t)) + else: + self.logger.debug("sync: %s -> %s" % (path_s, path_t)) + + lst_s = self.io_s.listdir(path_s) + try: + lst_t = self.io_t.listdir(path_t) + except OSError: + if self.dry_run: + lst_t = [] + else: + raise + + # in case io_s or io_t are case-insensitive, remove duplicate + # file names. + self._unique(lst_s, self.io_t.eq) + self._unique(lst_t, self.io_s.eq) + + for x in lst_s: + + abs_s = self.io_s.path.join(path_s, x) + isdir_s = self.isdir(self.io_s, abs_s) + + if not isdir_s and not self.io_s.path.isfile(abs_s): + self.logger.info("skipping non-file %s" % abs_s) + continue + + # This deletes x from lst_t. That enables us to simply + # iterate over lst_t later to find files to be deleted. + exists_t = self._pull(x, lst_t, self.io_t.eq) + abs_t = self.io_t.path.join(path_t, x) + if exists_t: + isdir_t = self.isdir(self.io_t, abs_t) + + if self.exclude(path, x, isdir_s): + self.logger.debug("exclude src %s/%s" % (path, x)) + if self.delete and self.delete_excluded and exists_t: + self.logger.log(NOTICE, "delete excluded %s/%s" % (path, x)) + if isdir_t: + todo.rmd.append(x) + else: + todo.unl.append(x) + continue + + # Here we know: src exists and is not excluded. + if isdir_s: + todo.dsc.append(x) + + if exists_t: + if isdir_s: + if not isdir_t: + todo.unl.append(x) + todo.mkd.append(x) + else: + if isdir_t: + todo.rmd.append(x) + todo.cpy.append(x) + reason[x] = "type" + else: + rsn = self.need_copy(abs_s, abs_t) + if rsn: + todo.unl.append(x) + todo.cpy.append(x) + reason[x] = "%s" % rsn + else: # not exists_t + if isdir_s: + todo.mkd.append(x) + else: + reason[x] = "new" + todo.cpy.append(x) + # for loop over lst_s ends + + if self.delete: + + # Anything now in lst_t didn't exist in src (see above) + for x in lst_t: + + abs_t = self.io_t.path.join(path_t, x) + isdir_t = self.isdir(self.io_t, abs_t) + + if self.exclude(path, x, isdir_t): + self.logger.debug("exclude tgt %s/%s" % (path, x)) + if not self.delete_excluded: + continue + + self.logger.info("delete %s/%s" % (path, x)) + if isdir_t: + todo.rmd.append(x) + else: + todo.unl.append(x) + + # From here on ACTIONS ARE CARRIED OUT + # First all remove actions, than mkdir and copy + for x in todo.rmd: + try: + self.logger.log(NOTICE, "rm -rf: %s/%s" % (path, x)) + self.rm_rf(self.io_t.path.join(path_t, x)) + except (OSError, IOError): + self.logger.exception("failed to rmdir %s" % x) + + for x in todo.unl: + try: + self.logger.log(NOTICE, "delete: %s/%s" % (path, x)) + if not self.dry_run: + self.io_t.unlink(self.io_t.path.join(path_t, x)) + except (OSError, IOError): + self.logger.exception("failed to unlink %s" % x) + + for x in todo.mkd: + try: + self.logger.log(NOTICE, "mkdir: %s/%s" % (path, x)) + if not self.dry_run: + self.io_t.mkdir(self.io_t.path.join(path_t, x)) + except (OSError, IOError): + self.logger.exception("failed to mkdir %s" % x) + self._pull(x, todo.dsc) + + for x in todo.cpy: + self.logger.log(NOTICE, "copy: %s/%s (reason: %s)" + % (path, x, reason[x])) + if not self.dry_run: + self.copy(self.io_s.path.join(path_s, x), + self.io_s.path.join(path_t, x)) + + # Finally, recurse. + for x in todo.dsc: + self.sync(self.io_s.path.join(path, x), False) + + if _top: + self.logger.info("sync finshed: %s -> %s" % (path_s, path_t)) + + +class RsyncSynchronizer(Synchronizer): + """ + Special Synchronzer class that uses rsyncmatch.GlobChain + for include/exclude logic. + """ + def __init__(self, *args, **kwargs): + Synchronizer.__init__(self, *args, **kwargs) + self.globchain = GlobChain() + + def exclude(self, dir, name, isdir): + path = self.io_s.path.join(dir, name) + if isdir: + path = path + "/" + + gl = self.globchain.match(path) + return gl == EXCLUDE Index: ftputil.py =================================================================== --- ftputil.py (revision 807:9b759d5191e2) +++ ftputil.py (revision 590:ccad912adb87) @@ -1,3 +1,3 @@ -# Copyright (C) 2002-2009, Stefan Schwarzer +# Copyright (C) 2002-2006, Stefan Schwarzer # All rights reserved. # @@ -89,10 +89,7 @@ import ftputil_version -# make exceptions available in this module for backwards compatibilty; -# you really should access them via the `ftp_error` module, not from here -from ftp_error import FTPError, FTPIOError, FTPOSError, \ - InaccessibleLoginDirError, InternalError, \ - ParserError, PermanentError, RootDirError, \ - TemporaryError, TimeShiftError +# make exceptions available in this module for backwards compatibilty +from ftp_error import * + # it's recommended to use the error classes via the `ftp_error` module; @@ -101,5 +98,4 @@ 'PermanentError', 'ParserError', 'FTPIOError', 'RootDirError', 'FTPHost'] - __version__ = ftputil_version.__version__ @@ -108,5 +104,5 @@ # `FTPHost` class with several methods similar to those of `os` -class FTPHost(object): +class FTPHost: """FTP host class.""" @@ -148,6 +144,4 @@ # associated `FTPHost` objects for data transfer self._children = [] - # only set if this instance represents an `_FTPFile` - self._file = None # now opened self.closed = False @@ -176,5 +170,9 @@ # this `FTPHost` object, use the same factory for this # `FTPHost` object's child sessions - factory = kwargs.pop('session_factory', ftplib.FTP) + if 'session_factory' in kwargs: + factory = kwargs['session_factory'] + del kwargs['session_factory'] + else: + factory = ftplib.FTP return ftp_error._try_with_oserror(factory, *args, **kwargs) @@ -228,28 +226,21 @@ host._file._open(effective_file, mode) if 'w' in mode: - # invalidate cache entry because size and timestamps will change self.stat_cache.invalidate(effective_path) return host._file - # make `open` an alias - open = file + def open(self, path, mode='r'): + # alias for `file` method + return self.file(path, mode) def close(self): """Close host connection.""" - if self.closed: - return - # close associated children - for host in self._children: - # children have a `_file` attribute which is an `_FTPFile` object - host._file.close() - host.close() - # now deal with ourself - try: + if not self.closed: + # close associated children + for host in self._children: + # only children have `_file` attributes + host._file.close() + host.close() + # now deal with our-self ftp_error._try_with_oserror(self._session.close) - finally: - # if something went wrong before, the host/session is - # probably defunct and subsequent calls to `close` won't - # help either, so we consider the host/session closed for - # practical purposes self.stat_cache.clear() self._children = [] @@ -257,43 +248,9 @@ def __del__(self): - # don't complain about lazy except clause - # pylint: disable-msg=W0702, W0704 try: self.close() except: - # we don't want warnings if the constructor failed + # we don't want warnings if the constructor did fail pass - - # - # setting a custom directory parser - # - def set_parser(self, parser): - """ - Set the parser for extracting stat results from directory - listings. - - The parser interface is described in the documentation, but - here are the most important things: - - - A parser should derive from `ftp_stat.Parser`. - - - The parser has to implement two methods, `parse_line` and - `ignores_line`. For the latter, there's a probably useful - default in the class `ftp_stat.Parser`. - - - `parse_line` should try to parse a line of a directory - listing and return a `ftp_stat.StatResult` instance. If - parsing isn't possible, raise `ftp_error.ParserError` with - a useful error message. - - - `ignores_line` should return a true value if the line isn't - assumed to contain stat information. - """ - # the cache contents, if any, aren't probably useful - self.stat_cache.clear() - # set the parser - self._stat._parser = parser - # just set a parser explicitly, don't allow "smart" switching anymore - self._stat._allow_parser_switching = False # @@ -371,7 +328,5 @@ Synchronize the local times of FTP client and server. This is necessary to let `upload_if_newer` and `download_if_newer` - work correctly. If `synchronize_times` isn't applicable - (see below), the time shift can still be set explicitly with - `set_time_shift`. + work correctly. This implementation of `synchronize_times` requires _all_ of @@ -382,5 +337,5 @@ current when `synchronize_times` is called. - The common usage pattern of `synchronize_times` is to call it + The usual usage pattern of `synchronize_times` is to call it directly after the connection is established. (As can be concluded from the points above, this requires write access @@ -414,8 +369,8 @@ # code directly because it might change) while True: - buffer_ = source.read(length) - if not buffer_: + buffer = source.read(length) + if not buffer: break - target.write(buffer_) + target.write(buffer) def __get_modes(self, mode): @@ -429,5 +384,5 @@ """ Copy a file from source to target. Which of both is a local - or a remote file, respectively, is determined by the arguments. + or a remote file, repectively, is determined by the arguments. """ source_mode, target_mode = self.__get_modes(mode) @@ -521,61 +476,5 @@ # - # helper methods to descend into a directory before executing a command - # - def _check_inaccessible_login_directory(self): - """ - Raise an `InaccessibleLoginDirError` exception if we can't - change to the login directory. This test is only reliable if - the current directory is the login directory. - """ - presumable_login_dir = self.getcwd() - # bail out with an internal error rather than modifying the - # current directory without hope of restoration - try: - self.chdir(presumable_login_dir) - except ftp_error.PermanentError: - # `old_dir` is an inaccessible login directory - raise ftp_error.InaccessibleLoginDirError( - "directory '%s' is not accessible" % presumable_login_dir) - - def _robust_ftp_command(self, command, path, descend_deeply=False): - """ - Run an FTP command on a path. The return value of the method - is the return value of the command. - - If `descend_deeply` is true (the default is false), descend - deeply, i. e. change the directory to the end of the path. - """ - # if we can't change to the yet-current directory, the code - # below won't work (see below), so in this case rather raise - # an exception than give wrong results - self._check_inaccessible_login_directory() - # Some FTP servers don't behave as expected if the directory - # portion of the path contains whitespace, some even yield - # strange results if the command isn't executed in the - # current directory. Therefore, change to the directory - # which contains the item to run the command on and invoke - # the command just there. - # remember old working directory - old_dir = self.getcwd() - try: - if descend_deeply: - # invoke the command in (not: on) the deepest directory - self.chdir(path) - # workaround for some servers that give recursive - # listings when called with a dot as path; see issue #33, - # http://ftputil.sschwarzer.net/trac/ticket/33 - return command(self, "") - else: - # invoke the command in the "next to last" directory - head, tail = self.path.split(path) - self.chdir(head) - return command(self, tail) - finally: - # restore the old directory - self.chdir(old_dir) - - # - # miscellaneous utility methods resembling functions in `os` + # miscellaneous utility methods resembling those in `os` # def getcwd(self): @@ -596,10 +495,5 @@ `os.mkdir`. """ - # ignore unused argument `mode` - # pylint: disable-msg=W0613 - def command(self, path): - """Callback function.""" - return ftp_error._try_with_oserror(self._session.mkd, path) - self._robust_ftp_command(command, path) + ftp_error._try_with_oserror(self._session.mkd, path) def makedirs(self, path, mode=None): @@ -610,6 +504,4 @@ `os.makedirs` but otherwise ignored. """ - # ignore unused argument `mode` - # pylint: disable-msg=W0613 path = self.path.abspath(path) directories = path.split(self.sep) @@ -617,6 +509,5 @@ # the "lowermost" directory for index in range(1, len(directories)): - # re-insert the separator which got lost by using `path.split` - next_directory = self.sep + self.path.join(*directories[:index+1]) + next_directory = self.path.join(*directories[:index+1]) try: self.mkdir(next_directory) @@ -643,8 +534,5 @@ raise ftp_error.PermanentError("directory '%s' not empty" % path) #XXX how will `rmd` work with links? - def command(self, path): - """Callback function.""" - ftp_error._try_with_oserror(self._session.rmd, path) - self._robust_ftp_command(command, path) + ftp_error._try_with_oserror(self._session.rmd, path) self.stat_cache.invalidate(path) @@ -654,12 +542,6 @@ # though `isfile` includes also links to files, `islink` # is needed to include links to directories - # if the path doesn't exist, let the removal command trigger - # an exception with a more appropriate error message - if self.path.isfile(path) or self.path.islink(path) or \ - not self.path.exists(path): - def command(self, path): - """Callback function.""" - ftp_error._try_with_oserror(self._session.delete, path) - self._robust_ftp_command(command, path) + if self.path.isfile(path) or self.path.islink(path): + ftp_error._try_with_oserror(self._session.delete, path) else: raise ftp_error.PermanentError("remove/unlink can only delete " @@ -668,11 +550,5 @@ def unlink(self, path): - """ - Remove the given file given by `path`. - - Raise a `PermanentError` if the path doesn't exist, raise a - `PermanentError`, but maybe raise other exceptions depending - on the state of the server (e. g. timeout). - """ + """Remove the given file.""" self.remove(path) @@ -689,11 +565,11 @@ `PermanentError`. - To distinguish between error situations, pass in a callable - for `onerror`. This callable must accept three arguments: - `func`, `path` and `exc_info`). `func` is a bound method - object, _for example_ `your_host_object.listdir`. `path` is - the path that was the recent argument of the respective method - (`listdir`, `remove`, `rmdir`). `exc_info` is the exception - info as it's got from `sys.exc_info`. + To distinguish between error situations and/or pass in a + callable for `onerror`. This callable must accept three + arguments: `func`, `path` and `exc_info`). `func` is a bound + method object, _for example_ `your_host_object.listdir`. + `path` is the path that was the recent argument of the + respective method (`listdir`, `remove`, `rmdir`). `exc_info` + is the exception info as it's got from `sys.exc_info`. Implementation note: The code is copied from `shutil.rmtree` @@ -703,22 +579,14 @@ # `shutil.rmtree` function if ignore_errors: - def new_onerror(*args): - """Do nothing.""" - # ignore unused arguments - # pylint: disable-msg=W0613 + def onerror(*args): pass elif onerror is None: - def new_onerror(*args): - """Re-raise exception.""" - # ignore unused arguments - # pylint: disable-msg=W0613 + def onerror(*args): raise - else: - new_onerror = onerror names = [] try: names = self.listdir(path) except ftp_error.PermanentError: - new_onerror(self.listdir, path, sys.exc_info()) + onerror(self.listdir, path, sys.exc_info()) for name in names: full_name = self.path.join(path, name) @@ -728,36 +596,18 @@ mode = 0 if stat.S_ISDIR(mode): - self.rmtree(full_name, ignore_errors, new_onerror) + self.rmtree(full_name, ignore_errors, onerror) else: try: self.remove(full_name) except ftp_error.PermanentError: - new_onerror(self.remove, full_name, sys.exc_info()) + onerror(self.remove, full_name, sys.exc_info()) try: self.rmdir(path) except ftp_error.FTPOSError: - new_onerror(self.rmdir, path, sys.exc_info()) + onerror(self.rmdir, path, sys.exc_info()) def rename(self, source, target): """Rename the source on the FTP host to target.""" - # the following code is in spirit similar to the code in the - # method `_robust_ftp_command`, though we don't do - # _everything_ imaginable - self._check_inaccessible_login_directory() - source_head, source_tail = self.path.split(source) - target_head, target_tail = self.path.split(target) - paths_contain_whitespace = (" " in source_head) or (" " in target_head) - if paths_contain_whitespace and source_head == target_head: - # both items are in the same directory - old_dir = self.getcwd() - try: - self.chdir(source_head) - ftp_error._try_with_oserror(self._session.rename, - source_tail, target_tail) - finally: - self.chdir(old_dir) - else: - # use straightforward command - ftp_error._try_with_oserror(self._session.rename, source, target) + ftp_error._try_with_oserror(self._session.rename, source, target) #XXX one could argue to put this method into the `_Stat` class, but @@ -770,19 +620,38 @@ # would cause a call of `(l)stat` and thus a call to `_dir`, # so we would end up with an infinite recursion - def _FTPHost_dir_command(self, path): - """Callback function.""" - lines = [] - def callback(line): - """Callback function.""" - lines.append(line) + lines = [] + def callback(line): + lines.append(line) + # see below for this decision logic + if " " not in path: + # use straight-forward approach, without changing directories ftp_error._try_with_oserror(self._session.dir, path, callback) - return lines - lines = self._robust_ftp_command(_FTPHost_dir_command, path, - descend_deeply=True) + else: + # remember old working directory + old_dir = self.getcwd() + # bail out with an internal error rather than modifying the + # current directory without hope of restoration + try: + self.chdir(old_dir) + except ftp_error.PermanentError: + # `old_dir` is an inaccessible login directory + raise ftp_error.InaccessibleLoginDirError( + "directory '%s' is not accessible" % old_dir) + # because of a bug in `ftplib` (or even in FTP servers?) + # the straight-forward code + # ftp_error._try_with_oserror(self._session.dir, path, callback) + # fails if some of the path components but the last contain + # whitespace; therefore, I change the current directory + # before listing in the "last" directory + try: + # invoke the listing in the "previous-to-last" directory + head, tail = self.path.split(path) + self.chdir(head) + ftp_error._try_with_oserror(self._session.dir, tail, callback) + finally: + # restore the old directory + self.chdir(old_dir) return lines - # the `listdir`, `lstat` and `stat` methods don't use - # `_robust_ftp_command` because they implicitly already use - # `_dir` which actually uses `_robust_ftp_command` def listdir(self, path): """ @@ -853,41 +722,7 @@ path = self.path.join(top, name) if not self.path.islink(path): - for item in self.walk(path, topdown, onerror): - yield item + for x in self.walk(path, topdown, onerror): + yield x if not topdown: yield top, dirs, nondirs - def chmod(self, path, mode): - """ - Change the mode of a remote `path` (a string) to the integer - `mode`. This integer uses the same bits as the mode value - returned by the `stat` and `lstat` commands. - - If something goes wrong, raise a `TemporaryError` or a - `PermanentError`, according to the status code returned by - the server. In particular, a non-existent path usually - causes a `PermanentError`. - """ - path = self.path.abspath(path) - def command(self, path): - """Callback function.""" - ftp_error._try_with_oserror(self._session.voidcmd, - "SITE CHMOD %s %s" % (oct(mode), path)) - self._robust_ftp_command(command, path) - self.stat_cache.invalidate(path) - - # - # context manager methods - # - def __enter__(self): - # return `self`, so it can be accessed as the variable - # component of the `with` statement. - return self - - def __exit__(self, exc_type, exc_val, exc_tb): - # we don't need the `exc_*` arguments here - # pylint: disable-msg=W0613 - self.close() - # be explicit - return False - Index: ftputil.txt =================================================================== --- ftputil.txt (revision 831:3335b387c7d2) +++ ftputil.txt (revision 602:923443c9cd90) @@ -1,7 +1,7 @@ -``ftputil`` -- a high-level FTP client library -============================================== - -:Version: 2.4.2 -:Date: 2009-11-12 +``ftputil`` - a high-level FTP client library +============================================= + +:Version: 2.2b +:Date: 2006-10-20 :Summary: high-level FTP client library for Python :Keywords: FTP, ``ftplib`` substitute, virtual filesystem, pure Python @@ -49,5 +49,5 @@ to `os.stat`_. Even `FTPHost.walk`_ and `FTPHost.path.walk`_ work. -.. _`os.stat`: http://www.python.org/doc/2.5/lib/os-file-dir.html#l2h-2698 +.. _`os.stat`: http://www.python.org/doc/current/lib/os-file-dir.html#l2h-1455 @@ -82,5 +82,5 @@ have many common methods like ``read``, ``readline``, ``readlines``, ``write``, ``writelines``, ``close`` and can do automatic line - ending conversions on the fly, i. e. text/binary mode). + ending conversions on the fly, i. e. text/binary mode) @@ -90,13 +90,11 @@ The exceptions are in the namespace of the ``ftp_error`` module, e. g. ``ftp_error.TemporaryError``. Getting the exception classes from the -"package module" ``ftputil`` is deprecated and will no longer be -supported in ``ftputil`` version 2.5. - -The exception classes are organized as follows:: +"package module" ``ftputil`` is deprecated. + +The exceptions are organized as follows:: FTPError FTPOSError(FTPError, OSError) PermanentError(FTPOSError) - CommandNotImplementedError(PermanentError) TemporaryError(FTPOSError) FTPIOError(FTPError) @@ -157,21 +155,13 @@ func(path, file, os=host) - to use the same code for both a local and remote file system. - Another similarity between ``OSError`` and ``FTPOSError`` is that - the latter holds the FTP server return code in the ``errno`` - attribute of the exception object and the error text in - ``strerror``. + to use the same code for a local and remote file system. Another + similarity between ``OSError`` and ``FTPOSError`` is that the latter + holds the FTP server return code in the ``errno`` attribute of the + exception object and the error text in ``strerror``. - ``PermanentError`` - is raised for 5xx return codes from the FTP server. This - corresponds to ``ftplib.error_perm`` (though ``PermanentError`` and - ``ftplib.error_perm`` are *not* identical). - -- ``CommandNotImplementedError`` - - indicates that an underlying command the code tries to use is not - implemented. For an example, see the description of the - `FTPHost.chmod`_ method. + is raised for 5xx return codes from the FTP server + (again, that's similar but *not* identical to ``ftplib.error_perm``). - ``TemporaryError`` @@ -212,6 +202,6 @@ 550 not_there: No such file or directory. - As you can see, both code snippets are similar. However, the error - codes aren't the same. + As you can see, both code snippets are similar. (However, the error + codes aren't the same.) - ``InternalError`` @@ -226,6 +216,5 @@ - The directory in which "you" are placed upon login is not - accessible, i. e. a ``chdir`` call with the directory as - argument would fail. + accessible, i. e. a ``chdir`` call fails. - You try to access a path which contains whitespace. @@ -261,8 +250,5 @@ ~~~~~~~~~~~~ -Basics -`````` - -``FTPHost`` instances can be generated with the following call:: +``FTPHost`` instances may be generated with the following call:: host = ftputil.FTPHost(host, user, password, account, @@ -270,18 +256,14 @@ The first four parameters are strings with the same meaning as for the -FTP class in the ``ftplib`` module. - -Session factories -````````````````` - -The keyword argument ``session_factory`` may be used to generate FTP -connections with other factories than the default ``ftplib.FTP``. For -example, the M2Crypto distribution uses a secure FTP class which is -derived from ``ftplib.FTP``. +FTP class in the ``ftplib`` module. The keyword argument +``session_factory`` may be used to generate FTP connections with other +factories than the default ``ftplib.FTP``. For example, the M2Crypto +distribution uses a secure FTP class which is derived from +``ftplib.FTP``. In fact, all positional and keyword arguments other than -``session_factory`` are passed to the factory to generate a new -background session. This happens for every remote file that is opened; -see below. +``session_factory`` are passed to the factory to generate a new background +session (which happens for every remote file that is opened; see +below). This functionality of the constructor also allows to wrap @@ -290,7 +272,7 @@ As an example, assume you want to connect to another than the default -port, but ``ftplib.FTP`` only offers this by means of its ``connect`` -method, not via its constructor. The solution is to use a wrapper -class:: +port but ``ftplib.FTP`` only offers this by means of its ``connect`` +method, but not via its constructor. The solution is to provide a +wrapper class:: import ftplib @@ -301,5 +283,5 @@ class MySession(ftplib.FTP): def __init__(self, host, userid, password, port): - """Act like ftplib.FTP's constructor but connect to another port.""" + """Act like ftplib.FTP's constructor but connect to other port.""" ftplib.FTP.__init__(self) self.connect(host, port) @@ -313,30 +295,7 @@ On login, the format of the directory listings (needed for stat'ing files and directories) should be determined automatically. If not, -please `file a bug report`_. - -.. _`file a bug report`: http://ftputil.sschwarzer.net/issuetrackernotes - -Support for the ``with`` statement -`````````````````````````````````` - -If you are sure that all the users of your code use at least Python -2.5, you can use Python's `with statement`_:: - - # not needed for Python 2.6 and later - from __future__ import with_statement - - import ftputil - - with ftputil.FTPHost(host, user, password) as host: - print host.listdir(host.curdir) - -After the ``with`` block, the ``FTPHost`` instance and the -associated FTP sessions will be closed automatically. - -If something goes wrong during the ``FTPHost`` construction or in the -body of the ``with`` statement, the instance is closed as well. -Exceptions will be propagated (as with ``try ... finally``). - -.. _`with statement`: http://www.python.org/dev/peps/pep-0343/ +please `file a bug`_. + +.. _`file a bug`: http://ftputil.sschwarzer.net/issuetrackernotes ``FTPHost`` attributes and methods @@ -349,8 +308,10 @@ are strings which denote the current and the parent directory on the - remote server. ``sep`` holds the path separator. Though `RFC 959`_ + remote server. sep identifies the path separator. Though `RFC 959`_ (File Transfer Protocol) notes that these values may depend on the - FTP server implementation, the Unix variants seem to work well in - practice, even for non-Unix servers. + FTP server implementation, the Unix counterparts seem to work well + in practice, even for non-Unix servers. + +.. _`RFC 959`: `RFC 959 - File Transfer Protocol (FTP)`_ Remote file system navigation @@ -373,15 +334,15 @@ copies a local source file (given by a filename, i. e. a string) - to the remote host under the name target. Both ``source`` and - ``target`` may be absolute paths or relative to their corresponding - current directory (on the local or the remote host, respectively). - The mode may be "" or "a" for ASCII uploads or "b" for binary - uploads. ASCII mode is the default, similar to regular local - file objects. + to the remote host under the name target. Both source and target + may be absolute paths or relative to their corresponding current + directory (on the local or the remote host, respectively). The + mode may be "" or "a" for ASCII uploads or "b" for binary uploads. + ASCII mode is the default (again, similar to regular local file + objects). - ``download(source, target, mode='')`` performs a download from the remote source to a target file. Both - ``source`` and ``target`` are strings. Most of the description of + source and target are strings. Additionally, the description of the upload method applies here, too. @@ -390,9 +351,9 @@ - ``upload_if_newer(source, target, mode='')`` - is similar to the ``upload`` method. The only difference is that the - upload is only invoked if the time of the last modification for the - source file is more recent than that of the target file or the - target doesn't exist at all. If an upload actually happened, the - return value is a true value, else a false value. + is similar to the upload method. The only difference is that the + upload is only invoked if the time of the last modification for + the source file is more recent than that of the target file, or + the target doesn't exist at all. If an upload actually happened, + the return value is a true value, else a false value. Note that this method only checks the existence and/or the @@ -410,5 +371,5 @@ newer modification time than the local file, and thus the transfer won't be repeated if ``upload_if_newer`` is used a second time. - There are at least two possibilities after a failed upload: + There are (at least) two possibilities after a failed upload: - use ``upload`` instead of ``upload_if_newer``, or @@ -417,6 +378,6 @@ use ``upload`` or ``upload_if_newer`` to transfer it again. - If it seems that a file is uploaded unnecessarily or not when it - should, read the subsection on `time shift`_ settings. + If it seems that a file is uploaded unnecessarily, read the + subsection on `time shift`_ settings. .. _`download_if_newer`: @@ -429,28 +390,17 @@ return value is a true value, else a false value. - If it seems that a file is downloaded unnecessarily or not when it - should, read the subsection on `time zone correction`_. + If it seems that a file is downloaded unnecessarily, read the + subsection on `time shift`_ settings. .. _`time shift`: -.. _`time zone correction`: Time zone correction ```````````````````` -If the client where ``ftputil`` runs and the server have a different -understanding of their local times, this has to be taken into account -for ``upload_if_newer`` and ``download_if_newer`` to work correctly. - -Note that even if the client and the server are in the same time zone -(or even on the same computer), the time shift value (see below) may -be different from zero. For example, my computer is set to use local -time whereas the server running on the very same host insists on using -UTC time. - .. _`set_time_shift`: - ``set_time_shift(time_shift)`` - sets the so-called time shift value, measured in seconds. The time + sets the so-called time shift value (measured in seconds). The time shift is the difference between the local time of the server and the local time of the client at a given moment, i. e. by definition @@ -460,12 +410,12 @@ time_shift = server_time - client_time - Setting this value is important for `upload_if_newer`_ and - `download_if_newer`_ to work correctly even if the time zone of the - FTP server differs from that of the client. Note that the time shift - value *can be negative*. + Setting this value is important if `upload_if_newer`_ and + `download_if_newer`_ should work correctly even if the time zone of + the FTP server differs from that of the client (where ``ftputil`` + runs). Note that the time shift value *can* be negative. If the time shift value is invalid, e. g. no multiple of a full hour - or its absolute value larger than 24 hours, a ``TimeShiftError`` is - raised. + or its absolute (unsigned) value larger than 24 hours, a + ``TimeShiftError`` is raised. See also `synchronize_times`_ for a way to set the time shift with a @@ -474,6 +424,6 @@ - ``time_shift()`` - returns the currently-set time shift value. See ``set_time_shift`` - above for its definition. + return the currently-set time shift value. See ``set_time_shift`` + (above) for its definition. .. _`synchronize_times`: @@ -483,5 +433,5 @@ synchronizes the local times of the server and the client, so that `upload_if_newer`_ and `download_if_newer`_ work as expected, even - if the client and the server use different time zones. For this + if the client and the server are in different time zones. For this to work, *all* of the following conditions must be true: @@ -492,6 +442,6 @@ If you can't fulfill these conditions, you can nevertheless set the - time shift value explicitly with `set_time_shift`_. Trying to call - ``synchronize_times`` if the above conditions aren't met results in + time shift value manually with `set_time_shift`_. Trying to call + ``synchronize_times`` if the above conditions aren't true results in a ``TimeShiftError`` exception. @@ -504,13 +454,14 @@ "intermediate" directories which don't already exist. The ``mode`` parameter is ignored; this is for compatibility with ``os.mkdir`` if - an ``FTPHost`` object is passed into a function instead of the - ``os`` module. See the explanation in the subsection `Exception - hierarchy`_. + an ``FTPHost`` object is passed into a function instead of the os + module (see the subsection on Python exceptions above for an + explanation). - ``makedirs(path, [mode])`` - works similar to ``mkdir`` (see above), but also makes intermediate - directories like ``os.makedirs``. The ``mode`` parameter is only - there for compatibility with ``os.makedirs`` and is ignored. + works similar to ``mkdir`` (see above, but also makes intermediate + directories, like ``os.makedirs``). The ``mode`` parameter is + only there for compatibility with ``os.makedirs`` and is + ignored. - ``rmdir(path)`` @@ -527,19 +478,24 @@ If ``ignore_errors`` is set to a true value, errors are ignored. If ``ignore_errors`` is a false value *and* ``onerror`` isn't - set, all exceptions occurring during the tree iteration and + set, all exceptions occuring during the tree iteration and processing are raised. These exceptions are all of type ``PermanentError``. - To distinguish between different kinds of errors, pass in a callable + To distinguish between error situations and/or pass in a callable for ``onerror``. This callable must accept three arguments: - ``func``, ``path`` and ``exc_info``. ``func`` is a bound method - object, *for example* ``your_host_object.listdir``. ``path`` is the - path that was the recent argument of the respective method + ``func``, ``path`` and ``exc_info``). ``func`` is a bound method + object, *for example* ``your_host_object.listdir``. ``path`` is + the path that was the recent argument of the respective method (``listdir``, ``remove``, ``rmdir``). ``exc_info`` is the exception - info as it is gotten from ``sys.exc_info``. + info as it is got from ``sys.exc_info``. The code of ``rmtree`` is taken from Python's ``shutil`` module and adapted for ``ftputil``. + **Note: I find this interface rather complicated and would like to + simplify it without making error handling too difficult. Possible + changes to ``rmtree`` will depend on the discussion between the + versions 2.1b and 2.1.** + Removing files and links ```````````````````````` @@ -547,5 +503,5 @@ - ``remove(path)`` - removes a file or link on the remote host, similar to ``os.remove``. + removes a file or link on the remote host (similar to ``os.remove``). - ``unlink(path)`` @@ -559,17 +515,16 @@ returns a list containing the names of the files and directories - in the given path, similar to ``os.listdir``. The special names + in the given path; similar to ``os.listdir``. The special names ``.`` and ``..`` are not in the list. -The methods ``lstat`` and ``stat`` (and some others) rely on the -directory listing format used by the FTP server. When connecting to a -host, ``FTPHost``'s constructor tries to guess the right format, which -succeeds in most cases. However, if you get strange results or +The methods ``lstat`` and ``stat`` (and others) rely on the directory +listing format used by the FTP server. When connecting to a host, +``FTPHost``'s constructor tries to guess the right format, which +mostly succeeds. However, if you get strange results or ``ParserError`` exceptions by a mere ``lstat`` call, please `file a -bug report`_. +bug`_. If ``lstat`` or ``stat`` yield wrong modification dates or times, look -at the methods that deal with time zone differences (`time zone -correction`_). +at the methods that deal with time zone differences (`time shift`_). .. _`FTPHost.lstat`: @@ -577,15 +532,16 @@ - ``lstat(path)`` - returns an object similar to that from ``os.lstat``. This is a - "tuple" with additional attributes; see the documentation of the - ``os`` module for details. - - The result is derived by parsing the output of a ``DIR`` command on - the server. Therefore, the result from ``FTPHost.lstat`` can not - contain more information than the received text. In particular: + returns an object similar that from ``os.lstat`` (a "tuple" with + additional attributes; see the documentation of the ``os`` module for + details). However, due to the nature of the application, there are + some important aspects to keep in mind: + + - The result is derived by parsing the output of a ``DIR`` command on + the server. Therefore, the result from ``FTPHost.lstat`` can not + contain more information than the received text. In particular: - User and group ids can only be determined as strings, not as numbers, and that only if the server supplies them. This is - usually the case with Unix servers but maybe not for other FTP + usually the case with Unix servers but may not be for other FTP server programs. @@ -599,30 +555,27 @@ information in the ``DIR`` output. - - Stat attributes that can't be determined at all are set to - ``None``. For example, a line of a directory listing may not - contain the date/time of a directory's last modification. + - Items that can't be determined at all are set to ``None``. - There's a special problem with stat'ing the root directory. (Stat'ing things *in* the root directory is fine though.) In this case, a ``RootDirError`` is raised. This has to do with the - algorithm used by ``(l)stat``, and I know of no approach which + algorithm used by ``(l)stat`` and I know of no approach which mends this problem. + +.. Currently, ``ftputil`` recognizes the common Unix-style and Microsoft/DOS-style directory formats. If you need to parse output from another server type, please write to the `ftputil mailing - list`_. You may consider to `write your own parser`_. + list`_. .. _`ftputil mailing list`: http://ftputil.sschwarzer.net/mailinglist -.. _`write your own parser`: `Writing directory parsers`_ .. _`FTPHost.stat`: - ``stat(path)`` - returns ``stat`` information also for files which are pointed to by a link. This method follows multiple links until a regular file or - directory is found. If an infinite link chain is encountered or the - target of the last link in the chain doesn't exist, a + directory is found. If an infinite link chain is encountered, a ``PermanentError`` is raised. @@ -654,8 +607,4 @@ walk(path, func, arg) -Like Python's counterparts under `os.path`_, ``ftputil``'s ``is...`` -methods return ``False`` if they can't find the path given by their -argument. - Local caching of file system information ```````````````````````````````````````` @@ -665,15 +614,15 @@ *each* call to ``lstat``, ``stat``, ``exists``, ``getmtime`` etc. would require to fetch a directory listing from the server, which can -make the program *very* slow. This effect is more pronounced for +make the program very slow. This effect is more pronounced for operations which mostly scan the file system rather than transferring file data. -For this reason, ``ftputil`` by default saves the results from -directory listings locally and reuses those results. This reduces +For this reason, ``ftputil`` by default saves (caches) the results +from directory listings locally and reuses those results. This reduces network accesses and so speeds up the software a lot. However, since data is more rarely fetched from the server, the risk of obsolete data also increases. This will be discussed below. -Caching can be controlled -- if necessary at all -- via the +Caching can - if necessary at all - be controlled via the ``stat_cache`` object in an ``FTPHost``'s namespace. For example, after calling @@ -687,24 +636,24 @@ While ``ftputil`` usually manages the cache quite well, there are two -possible reasons that may suggest modifying cache parameters. -The first is when the number of possible entries is too low. You may -notice that when you are processing very large directories, e. g. -containing more than 1000 directories or files, and the program -becomes much slower than before. It's common for code to read a -directory with ``listdir`` and then process the found directories and -files. For this application, it's a good rule of thumb to set the -cache size to somewhat more than the number of directory entries -fetched with ``listdir``. This is done by the ``resize`` method:: +possible reasons for modifying cache parameters. The first is when the +number of possible entries is too low. You may notice that when you +are processing very large directories (e. g. above 1000 directories or +files) and the program becomes much slower than before. It's common +for code to read a directory with ``listdir`` and then process the +found directories and files. For this application, it's a good rule of +thumb to set the cache size to somewhat more than the number of +directory entries fetched with ``listdir``. This is done by the +``resize`` method:: host.stat_cache.resize(2000) -where the argument is the maximum number of ``lstat`` results to store +where the argument is the maximal number of ``lstat`` results to store (the default is 1000). Note that each path on the server, e. g. -"/home/schwa/some_dir", corresponds to a single cache entry. Methods +"/home/schwa/some_dir", corresponds to a single cache entry. (Methods like ``exists`` or ``getmtime`` all derive their results from a -previously fetched ``lstat`` result. +previously fetched ``lstat`` result.) The value 2000 above means that the cache will hold at most 2000 -entries. If more are about to be stored, the entries which haven't +entries. If more are about to be stored, the entries which have not been used for the longest time will be deleted to make place for newer entries. @@ -720,5 +669,5 @@ changes are handled transparently; the path will be deleted from the cache. A different matter are changes unknown to the ``FTPHost`` -object which inspects its cache. Obviously, for example, these are +object which reads its cache. Obviously, for example, these are changes by programs running on the remote host. On the other hand, cache inconsistencies can also occur if two ``FTPHost`` objects change @@ -732,8 +681,8 @@ host2.remove("some_file") # `host1` will still see the obsolete cache entry! - print host1.stat("some_file") + print stat_result1 # will raise an exception since an `FTPHost` object # knows of its own changes - print host2.stat("some_file") + print stat_result2 finally: host1.close() @@ -744,8 +693,8 @@ to be very error-prone. For example, it won't help with different processes using ``ftputil``. So, if you have to deal with concurrent -write/read accesses to a server, you have to handle them explicitly. - -The most useful tool for this is the ``invalidate`` method. In the -example above, it could be used like this:: +write accesses to a server, you have to handle them explicitly. + +The most useful tool for this probably is the ``invalidate`` method. +In the example above, it could be used as:: host1 = ftputil.FTPHost(server, user1, password1) @@ -760,8 +709,8 @@ host1.stat_cache.invalidate(absolute_path) # will now raise an exception as it should - print host1.stat("some_file") + print stat_result1 # would raise an exception since an `FTPHost` object # knows of its own changes, even without `invalidate` - print host2.stat("some_file") + print stat_result2 finally: host1.close() @@ -771,9 +720,8 @@ directory, a file or a link. -By default, the cache entries (if not replaced by newer ones) are -stored for an infinite time. That is, if you start your Python process -using ``ftputil`` and let it run for three days a stat call may still -access cache data that old. To avoid this, you can set the ``max_age`` -attribute:: +By default, the cache entries are stored indefinetely, i. e. if you +start your Python process using ``ftputil`` and let it run for three +days a stat call may still access cache data that old. To avoid this, +you can set the ``max_age`` attribute:: host = ftputil.FTPHost(server, user, password) @@ -782,10 +730,10 @@ This sets the maximum age of entries in the cache to an hour. This means any entry older won't be retrieved from the cache but its data -instead fetched again from the remote host and then again stored for -up to an hour. To reset `max_age` to the default of unlimited age, +instead fetched again from the remote host (and then again stored for +up to an hour). To reset `max_age` to the default of unlimited age, i. e. cache entries never expire, use ``None`` as value. -If you are certain that the cache will be in the way, you can disable -and later re-enable it completely with ``disable`` and ``enable``:: +If you are certain that the cache is in the way, you can disable and +later re-enable it completely with ``disable`` and ``enable``:: host = ftputil.FTPHost(server, user, password) @@ -816,9 +764,9 @@ - ``walk(top, topdown=True, onerror=None)`` - iterates over a directory tree, similar to `os.walk`_. Actually, - ``FTPHost.walk`` uses the code from Python with just the necessary - modifications, so see the linked documentation. - -.. _`os.walk`: http://www.python.org/doc/2.5/lib/os-file-dir.html#l2h-2707 + iterates over a directory tree, similar to `os.walk`_ in Python 2.3 + and above. Actually, ``FTPHost.walk`` uses the code from Python with + just the necessary modifications, so see the linked documentation. + +.. _`os.walk`: http://docs.python.org/lib/os-file-dir.html#l2h-1638 .. _`FTPHost.path.walk`: @@ -827,6 +775,5 @@ Similar to ``os.path.walk``, the ``walk`` method in - `FTPHost.path`_ can be used, though ``FTPHost.walk`` is probably - easier to use. + `FTPHost.path`_ can be used. Other methods @@ -842,44 +789,4 @@ renames the source file (or directory) on the FTP server. - -.. _`FTPHost.chmod`: - -- ``chmod(path, mode)`` - - sets the access mode (permission flags) for the given path. The mode - is an integer as returned for the mode by the ``stat`` and ``lstat`` - methods. Be careful: Usually, mode values are written as octal - numbers, for example 0755 to make a directory readable and writable - for the owner, but not writable for the group and others. If you - want to use such octal values, rely on Python's support for them:: - - host.chmod("some_directory", 0755) - - *Note the leading zero.* - - Not all FTP servers support the ``chmod`` command. In case of - an exception, how do you know if the path doesn't exist or if - the command itself is invalid? If the FTP server complies with - `RFC 959`_, it should return a status code 502 if the ``SITE CHMOD`` - command isn't allowed. ``ftputil`` maps this special error - response to a ``CommandNotImplementedError`` which is derived from - ``PermanentError``. - - So you need to code like this:: - - host = ftputil.FTPHost(server, user, password) - try: - host.chmod("some_file", 0644) - except ftp_error.CommandNotImplementedError: - # chmod not supported - ... - except ftp_error.PermanentError: - # possibly a non-existent file - ... - - Because the ``CommandNotImplementedError`` is more specific, you - have to test for it first. - -.. _`RFC 959`: `RFC 959 - File Transfer Protocol (FTP)`_ - ``copyfileobj(source, target, length=64*1024)`` @@ -892,18 +799,4 @@ and use of remote file-like objects. -.. _`set_parser`: - -- ``set_parser(parser)`` - - sets a custom parser for FTP directories. Note that you have to pass - in a parser *instance*, not the class. - - An `extra section`_ shows how to write own parsers if the default - parsers in ``ftputil`` don't work for you. Possibly you are lucky - and someone has already written a parser you can use. Please ask on - the `mailing list`_. - -.. _`extra section`: `Writing directory parsers`_ - File-like objects @@ -913,9 +806,6 @@ ~~~~~~~~~~~~ -Basics -`````` - -``FTPFile`` objects are returned by a call to ``FTPHost.file`` or -``FTPHost.open``, never use the constructor directly. +``FTPFile`` objects are returned by a call to ``FTPHost.file`` (or +``FTPHost.open``). - ``FTPHost.file(path, mode='r')`` @@ -930,31 +820,4 @@ is an alias for ``file`` (see above). - -Support for the ``with`` statement -`````````````````````````````````` - -If you are sure that all the users of your code use at least Python -2.5, you can use Python's `with statement`_ with the ``FTPFile`` -constructor:: - - # not needed for Python 2.6 and later - from __future__ import with_statement - - import ftputil - - # get an ``FTPHost`` object from somewhere - ... - - with host.file("new_file", "w") as f: - f.write("This is some text.") - -At the end of the ``with`` block, the file will be closed -automatically. - -If something goes wrong during the construction of the file or in the -body of the ``with`` statement, the file will be closed as well. -Exceptions will be propagated as with ``try ... finally``. - -.. _`with statement`: http://www.python.org/dev/peps/pep-0343/ Attributes and methods @@ -974,7 +837,6 @@ and the attribute ``closed`` have the same semantics as for file -objects of a local disk file system. The iterator protocol is -supported as well, i. e. you can use a loop to read a file line by -line:: +objects of a local disk file system. The iterator protocol is also +supported, i. e. you can use a loop to read a file line by line:: host = ftputil.FTPHost(...) @@ -985,9 +847,6 @@ input_file.close() -This feature obsoletes the ``xreadlines`` method which is deprecated -and will be removed in ``ftputil`` version 2.5. - For more on file objects, see the section `File objects`_ in the -Python Library Reference. +Library Reference. .. _`file objects`: http://www.python.org/doc/current/lib/bltin-file-objects.html @@ -995,122 +854,4 @@ Note that ``ftputil`` supports both binary mode and text mode with the appropriate line ending conversions. - - -Writing directory parsers -------------------------- - -``ftputil`` recognizes the two most widely-used FTP directory formats -(Unix and MS style) and adjusts itself automatically. However, if your -server uses a format which is different from the two provided by -``ftputil``, you can plug in a custom parser and have it used by -a single method call. - -For this, you need to write a parser class by inheriting from the -class ``Parser`` in the ``ftp_stat`` module. Here's an example:: - - from ftputil import ftp_error - from ftputil import ftp_stat - - class XyzParser(ftp_stat.Parser): - """ - Parse the default format of the FTP server of the XYZ - corporation. - """ - def parse_line(self, line, time_shift=0.0): - """ - Parse a `line` from the directory listing and return a - corresponding `StatResult` object. If the line can't - be parsed, raise `ftp_error.ParserError`. - - The `time_shift` argument can be used to fine-tune the - parsing of dates and times. See the class - `ftp_stat.UnixParser` for an example. - """ - # split the `line` argument and examine it further; if - # something goes wrong, raise an `ftp_error.ParserError` - ... - # make a `StatResult` object from the parts above - stat_result = ftp_stat.StatResult(...) - # `_st_name` and `_st_target` are optional - stat_result._st_name = ... - stat_result._st_target = ... - return stat_result - - # define `ignores_line` only if the default in the base class - # doesn't do enough! - def ignores_line(self, line): - """ - Return a true value if the line should be ignored. For - example, the implementation in the base class handles - lines like "total 17". On the other hand, if the line - should be used for stat'ing, return a false value. - """ - is_total_line = super(XyzParser, self).ignores_line(line) - my_test = ... - return is_total_line or my_test - -A ``StatResult`` object is similar to the value returned by -`os.stat`_ and is usually built with statements like - -:: - - stat_result = StatResult( - (st_mode, st_ino, st_dev, st_nlink, st_uid, - st_gid, st_size, st_atime, st_mtime, st_ctime) ) - stat_result._st_name = ... - stat_result._st_target = ... - -with the arguments of the ``StatResult`` constructor described in -the following table. - -===== ========== ============ =============== ======================= -Index Attribute os.stat type StatResult type Notes -===== ========== ============ =============== ======================= -0 st_mode int int -1 st_ino long long -2 st_dev long long -3 st_nlink int int -4 st_uid int str usually only available as string -5 st_gid int str usually only available as string -6 st_size long long -7 st_atime int/float float -8 st_mtime int/float float -9 st_ctime int/float float -\- _st_name \- str file name without directory part -\- _st_target \- str link target -===== ========== ============ =============== ======================= - -If you can't extract all the desirable data from a line (for -example, the MS format doesn't contain any information about the -owner of a file), set the corresponding values in the ``StatResult`` -instance to ``None``. - -Parser classes can use several helper methods which are defined in -the class ``Parser``: - -- ``parse_unix_mode`` parses strings like "drwxr-xr-x" and returns - an appropriate ``st_mode`` value. - -- ``parse_unix_time`` returns a float number usable for the - ``st_...time`` values by parsing arguments like "Nov"/"23"/"02:33" or - "May"/"26"/"2005". Note that the method expects the timestamp string - already split at whitespace. - -- ``parse_ms_time`` parses arguments like "10-23-01"/"03:25PM" and - returns a float number like from ``time.mktime``. Note that the - method expects the timestamp string already split at whitespace. - -Additionally, there's an attribute ``_month_numbers`` which maps -lowercase three-letter month abbreviations to integers. - -For more details, see the two "standard" parsers ``UnixParser`` and -``MSParser`` in the module ``ftp_stat.py``. - -To actually *use* the parser, call the method `set_parser`_ of the -``FTPHost`` instance. - -If you can't write a parser or don't want to, please ask on the -`ftputil mailing list`_. Possibly someone has already written a parser -for your server or can help to do it. @@ -1135,9 +876,4 @@ subscribe or read the archives. -Though you can *technically* post without subscribing first I can't -recommend that: The mails from non-subscribers have to be approved by -me and because the arriving mails contain *lots* of spam, I rarely go -through this bunch of mails. - I found a bug! What now? ~~~~~~~~~~~~~~~~~~~~~~~~ @@ -1150,5 +886,5 @@ Please see http://ftputil.sschwarzer.net/issuetrackernotes for guidelines on entering a bug in ``ftputil``'s ticket system. If you -are unsure if the behaviour you found is a bug or not, you should write +are unsure if the behaviour you found is a bug or not, you can write to the `ftputil mailing list`_. In *either* case you *must not* include confidential information (user id, password, file names, etc.) @@ -1158,5 +894,5 @@ ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -``ftputil`` has no *built-in* SSL support. On the other hand, +``ftputil`` has no built-in SSL support. On the other hand, you can use M2Crypto_ (in the source code archive, look for the file ``M2Crypto/ftpslib.py``) which has a class derived from @@ -1176,9 +912,7 @@ """ ftpslib.FTP_TLS.__init__(self) - # do anything necessary to set up the SSL connection - ... self.connect(host, port) self.login(userid, password) - ... + # do anything necessary to set up the SSL connection # note the `session_factory` parameter @@ -1228,6 +962,6 @@ unnecessarily, or not when it should. This can happen when the FTP server is in a different time zone than the client on which -``ftputil`` runs. Please see the section on `time zone correction`_. -It may even be sufficient to call `synchronize_times`_. +``ftputil`` runs. Please see the the section on setting the +`time shift`_. It may even be sufficient to call `synchronize_times`_. Wrong dates or times when stat'ing on a server @@ -1235,34 +969,4 @@ Please see the previous tip. - -I tried to upload or download a file and it's corrupt -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -Perhaps you used the upload or download methods without a ``mode`` -argument. For compatibility with Python's code for local file systems, -``ftputil`` defaults to ASCII/text mode which will try to convert -presumable line endings and thus corrupt binary files. Pass "b" as the -``mode`` argument (see `Uploading and downloading files`_). - -When I use ``ftputil``, all I get is a ``ParserError`` exception -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -The FTP server you connect to uses a directory format that -``ftputil`` doesn't understand. You can either write and -`plug in an own parser`_, or preferably ask on the `mailing list`_ for -help. - -.. _`plug in an own parser`: `Writing directory parsers`_ - -``isdir``, ``isfile`` or ``islink`` incorrectly return ``False`` -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -Like Python's counterparts under `os.path`_, ``ftputil``'s methods -return ``False`` if they can't find the given path. - -Probably you used ``listdir`` on a directory and called ``is...()`` on -the returned names. But if the argument for ``listdir`` wasn't the -current directory, the paths won't be found and so all ``is...()`` -variants will return ``False``. I don't find an answer to my problem in this document @@ -1290,11 +994,10 @@ - Until now, I haven't paid attention to thread safety. In principle, at least, different ``FTPFile`` objects should be usable in different - threads. If in doubt if your approach will work, ask on the mailing - list. - -- ``FTPFile`` objects in text mode *may not* support charsets with - more than one byte per character. Please e-mail your experiences to - the mailing list (see above), if you work with multibyte text - streams in FTP sessions. + threads. + +- ``FTPFile`` objects in text mode *may not* support charsets with more + than one byte per character. Please email me your experiences + (address above), if you work with multibyte text streams in FTP + sessions. - Currently, it is not possible to continue an interrupted upload or @@ -1310,11 +1013,11 @@ If not overwritten via installation options, the ``ftputil`` files -reside in the ``ftputil`` package. The documentation in -`reStructuredText`_ and in HTML format is in the same directory. - -.. _`reStructuredText`: http://docutils.sourceforge.net/rst.html +reside in the ``ftputil`` package. The documentation (in +`reStructured Text`_ and in HTML format) is in the same directory. + +.. _`reStructured Text`: http://docutils.sourceforge.net/rst.html The files ``_test_*.py`` and ``_mock_ftplib.py`` are for unit-testing. -If you only *use* ``ftputil``, i. e. *don't* modify it, you can +If you only *use* ``ftputil`` (i. e. *don't* modify it), you can delete these files. @@ -1336,13 +1039,8 @@ -Authors -------- - -``ftputil`` is written by Stefan Schwarzer -, in part based on suggestions -from users. - -The ``lrucache`` module is written by Evan Prodromou -. +Author +------ + +``ftputil`` is written by Stefan Schwarzer . Feedback is appreciated. :-) Index: ftputil_version.py =================================================================== --- ftputil_version.py (revision 831:3335b387c7d2) +++ ftputil_version.py (revision 602:923443c9cd90) @@ -1,3 +1,3 @@ -# Copyright (C) 2006-2008, Stefan Schwarzer +# Copyright (C) 2006, Stefan Schwarzer # All rights reserved. # @@ -34,7 +34,6 @@ import sys - # ftputil version number -__version__ = '2.4.2' +__version__ = '2.2b' _ftputil_version = __version__ Index: lrucache.py =================================================================== --- lrucache.py (revision 812:d16f222c8d44) +++ lrucache.py (revision 574:71bf82018417) @@ -9,5 +9,5 @@ # # The original file is available at -# http://pypi.python.org/pypi/lrucache/0.2 . +# http://cheeseshop.python.org/pypi/lrucache/0.2 . # arch-tag: LRU cache main module @@ -43,7 +43,5 @@ from heapq import heappush, heappop, heapify -# the suffix after the hyphen denotes modifications by the -# ftputil project with respect to the original version -__version__ = "0.2-2" +__version__ = "0.2" __all__ = ['CacheKeyError', 'LRUCache', 'DEFAULT_SIZE'] __docformat__ = 'reStructuredText en' @@ -103,5 +101,5 @@ """Record of a cached value. Not for public consumption.""" - def __init__(self, key, obj, timestamp, sort_key): + def __init__(self, key, obj, timestamp): object.__init__(self) self.key = key @@ -109,8 +107,7 @@ self.atime = timestamp self.mtime = self.atime - self._sort_key = sort_key def __cmp__(self, other): - return cmp(self._sort_key, other._sort_key) + return cmp(self.atime, other.atime) def __repr__(self): @@ -121,24 +118,15 @@ def __init__(self, size=DEFAULT_SIZE): # Check arguments - if size < 0: - raise ValueError("cache size (%d) mustn't be negative" % size) + if size <= 0: + raise ValueError, size + elif type(size) is not type(0): + raise TypeError, size object.__init__(self) self.__heap = [] self.__dict = {} + self.size = size """Maximum size of the cache. If more than 'size' elements are added to the cache, the least-recently-used ones will be discarded.""" - self.size = size - self.__counter = 0 - - def _sort_key(self): - """Return a new integer value upon every call. - - Cache nodes need a monotonically increasing time indicator. - time.time() and time.clock() don't guarantee this in a - platform-independent way. - """ - self.__counter += 1 - return self.__counter def __len__(self): @@ -149,24 +137,16 @@ def __setitem__(self, key, obj): - if self.size == 0: - # can't store anything - return if self.__dict.has_key(key): node = self.__dict[key] - # update node object in-place node.obj = obj node.atime = time.time() node.mtime = node.atime - node._sort_key = self._sort_key() heapify(self.__heap) else: - # size of the heap can be at most the value of - # self.size because __setattr__ decreases the cache - # size if the new size value is smaller; so we don't - # need a loop _here_ - if len(self.__heap) == self.size: + # size may have been reset, so we loop + while len(self.__heap) >= self.size: lru = heappop(self.__heap) del self.__dict[lru.key] - node = self.__Node(key, obj, time.time(), self._sort_key()) + node = self.__Node(key, obj, time.time()) self.__dict[key] = node heappush(self.__heap, node) @@ -177,7 +157,5 @@ else: node = self.__dict[key] - # update node object in-place node.atime = time.time() - node._sort_key = self._sort_key() heapify(self.__heap) return node.obj @@ -204,6 +182,4 @@ # automagically shrink heap on resize if name == 'size': - if value < 0: - raise ValueError("cache size (%d) mustn't be negative" % size) while len(self.__heap) > value: lru = heappop(self.__heap) Index: ylintrc =================================================================== --- pylintrc (revision 714:5353ec9a050e) +++ (revision ) @@ -1,310 +1,0 @@ -# lint Python modules using external checkers. -# -# This is the main checker controling the other ones and the reports -# generation. It is itself both a raw checker and an astng checker in order -# to: -# * handle message activation / deactivation at the module level -# * handle some basic but necessary stats'data (number of classes, methods...) -# -[MASTER] - -# Specify a configuration file. -#rcfile= - -# Python code to execute, usually for sys.path manipulation such as -# pygtk.require(). -#init-hook= - -# Profiled execution. -profile=no - -# Add to the black list. It should be a base name, not a -# path. You may set this option multiple times. -ignore=CVS - -# Pickle collected data for later comparisons. -persistent=yes - -# Set the cache size for astng objects. -cache-size=500 - -# List of plugins (as comma separated values of python modules names) to load, -# usually to register additional checkers. -load-plugins= - - -[MESSAGES CONTROL] - -# Enable only checker(s) with the given id(s). This option conflicts with the -# disable-checker option -#enable-checker= - -# Enable all checker(s) except those with the given id(s). This option -# conflicts with the enable-checker option -#disable-checker= - -# Enable all messages in the listed categories. -#enable-msg-cat= - -# Disable all messages in the listed categories. -#disable-msg-cat= - -# Enable the message(s) with the given id(s). -#enable-msg= - -# Disable the message(s) with the given id(s). -# W0142: Used * or ** magic -disable-msg=W0142 - - -[REPORTS] - -# set the output format. Available formats are text, parseable, colorized, msvs -# (visual studio) and html -output-format=text - -# Include message's id in output -include-ids=yes - -# Put messages in a separate file for each module / package specified on the -# command line instead of printing them on stdout. Reports (if any) will be -# written in a file name "pylint_global.[txt|html]". -files-output=no - -# Tells wether to display a full report or only the messages -reports=yes - -# Python expression which should return a note less than 10 (10 is the highest -# note).You have access to the variables errors warning, statement which -# respectivly contain the number of errors / warnings messages and the total -# number of statements analyzed. This is used by the global evaluation report -# (R0004). -evaluation=10.0 - ((float(5 * error + warning + refactor + convention) / statement) * 10) - -# Add a comment according to your evaluation note. This is used by the global -# evaluation report (R0004). -comment=yes - -# Enable the report(s) with the given id(s). -#enable-report= - -# Disable the report(s) with the given id(s). -#disable-report= - - -# checks for : -# * doc strings -# * modules / classes / functions / methods / arguments / variables name -# * number of arguments, local variables, branchs, returns and statements in -# functions, methods -# * required module attributes -# * dangerous default values as arguments -# * redefinition of function / method / class -# * uses of the global statement -# -[BASIC] - -# Required attributes for module, separated by a comma -required-attributes= - -# Regular expression which should only match functions or classes name which do -# not require a docstring -no-docstring-rgx=__.*__ - -# Regular expression which should only match correct module names -module-rgx=(([a-z_][a-z0-9_]*)|([A-Z][a-zA-Z0-9]+))$ - -# Regular expression which should only match correct module level names -const-rgx=(([A-Z_][A-Z1-9_]*)|(__.*__))$ - -# Regular expression which should only match correct class names -class-rgx=[A-Z_][a-zA-Z0-9]+$ - -# Regular expression which should only match correct function names -function-rgx=[a-z_][a-z0-9_]{2,30}$ - -# Regular expression which should only match correct method names -method-rgx=[a-z_][a-z0-9_]{2,30}$ - -# Regular expression which should only match correct instance attribute names -attr-rgx=[a-z_][a-z0-9_]{2,30}$ - -# Regular expression which should only match correct argument names -argument-rgx=[a-z_][a-z0-9_]{2,30}$ - -# Regular expression which should only match correct variable names -variable-rgx=[a-z_][a-z0-9_]{2,30}$ - -# Regular expression which should only match correct list comprehension / -# generator expression variable names -inlinevar-rgx=[A-Za-z_][A-Za-z0-9_]*$ - -# Good variable names which should always be accepted, separated by a comma -good-names=i,j,k,ex,Run,_ - -# Bad variable names which should always be refused, separated by a comma -bad-names=foo,bar,baz,toto,tutu,tata - -# List of builtins function names that should not be used, separated by a comma -bad-functions=map,filter,apply,input - - -# try to find bugs in the code using type inference -# -[TYPECHECK] - -# Tells wether missing members accessed in mixin class should be ignored. A -# mixin class is detected if its name ends with "mixin" (case insensitive). -ignore-mixin-members=yes - -# List of classes names for which member attributes should not be checked -# (useful for classes with attributes dynamicaly set). -ignored-classes=SQLObject - -# When zope mode is activated, consider the acquired-members option to ignore -# access to some undefined attributes. -zope=no - -# List of members which are usually get through zope's acquisition mecanism and -# so shouldn't trigger E0201 when accessed (need zope=yes to be considered). -acquired-members=REQUEST,acl_users,aq_parent - - -# checks for -# * unused variables / imports -# * undefined variables -# * redefinition of variable from builtins or from an outer scope -# * use of variable before assigment -# -[VARIABLES] - -# Tells wether we should check for unused import in __init__ files. -init-import=no - -# A regular expression matching names used for dummy variables (i.e. not used). -dummy-variables-rgx=_|dummy - -# List of additional names supposed to be defined in builtins. Remember that -# you should avoid to define new builtins when possible. -additional-builtins= - - -# checks for sign of poor/misdesign: -# * number of methods, attributes, local variables... -# * size, complexity of functions, methods -# -[DESIGN] - -# Maximum number of arguments for function / method -max-args=5 - -# Maximum number of locals for function / method body -max-locals=15 - -# Maximum number of return / yield for function / method body -max-returns=6 - -# Maximum number of branch for function / method body -max-branchs=12 - -# Maximum number of statements in function / method body -max-statements=50 - -# Maximum number of parents for a class (see R0901). -max-parents=7 - -# Maximum number of attributes for a class (see R0902). -max-attributes=7 - -# Minimum number of public methods for a class (see R0903). -min-public-methods=2 - -# Maximum number of public methods for a class (see R0904). -max-public-methods=20 - - -# checks for : -# * methods without self as first argument -# * overridden methods signature -# * access only to existant members via self -# * attributes not defined in the __init__ method -# * supported interfaces implementation -# * unreachable code -# -[CLASSES] - -# List of interface methods to ignore, separated by a comma. This is used for -# instance to not check methods defines in Zope's Interface base class. -ignore-iface-methods=isImplementedBy,deferred,extends,names,namesAndDescriptions,queryDescriptionFor,getBases,getDescriptionFor,getDoc,getName,getTaggedValue,getTaggedValueTags,isEqualOrExtendedBy,setTaggedValue,isImplementedByInstancesOf,adaptWith,is_implemented_by - -# List of method names used to declare (i.e. assign) instance attributes. -defining-attr-methods=__init__,__new__,setUp - - -# checks for -# * external modules dependencies -# * relative / wildcard imports -# * cyclic imports -# * uses of deprecated modules -# -[IMPORTS] - -# Deprecated modules which should not be used, separated by a comma -deprecated-modules=regsub,string,TERMIOS,Bastion,rexec - -# Create a graph of every (i.e. internal and external) dependencies in the -# given file (report R0402 must not be disabled) -import-graph= - -# Create a graph of external dependencies in the given file (report R0402 must -# not be disabled) -ext-import-graph= - -# Create a graph of internal dependencies in the given file (report R0402 must -# not be disabled) -int-import-graph= - - -# checks for : -# * unauthorized constructions -# * strict indentation -# * line length -# * use of <> instead of != -# -[FORMAT] - -# Maximum number of characters on a single line. -max-line-length=80 - -# Maximum number of lines in a module -max-module-lines=1000 - -# String used as indentation unit. This is usually " " (4 spaces) or "\t" (1 -# tab). -indent-string=' ' - - -# checks for similarities and duplicated code. This computation may be -# memory / CPU intensive, so you should disable it if you experiments some -# problems. -# -[SIMILARITIES] - -# Minimum lines number of a similarity. -min-similarity-lines=4 - -# Ignore comments when computing similarities. -ignore-comments=yes - -# Ignore docstrings when computing similarities. -ignore-docstrings=yes - - -# checks for: -# * warning notes in the code like FIXME, XXX -# * PEP 263: source code with non ascii character but no encoding declaration -# -[MISCELLANEOUS] - -# List of note tags to take in consideration, separated by a comma. -notes=FIXME,XXX,TODO Index: andbox/ftpsync-0.1/LICENSE =================================================================== --- sandbox/ftpsync-0.1/LICENSE (revision 607:f2ab969f29b5) +++ (revision ) @@ -1,30 +1,0 @@ -# Copyright (C) 2006 Martin Wilck -# All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are -# met: -# -# - Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# -# - Redistributions in binary form must reproduce the above copyright -# notice, this list of conditions and the following disclaimer in the -# documentation and/or other materials provided with the distribution. -# -# - Neither the name of the above author nor the names of the -# contributors to the software may be used to endorse or promote -# products derived from this software without specific prior written -# permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -# ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR -# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -# LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -# NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. Index: andbox/ftpsync-0.1/README =================================================================== --- sandbox/ftpsync-0.1/README (revision 607:f2ab969f29b5) +++ (revision ) @@ -1,47 +1,0 @@ -Copyright (c) Martin Wilck 2006 - -See the file LICENSE for copyright information. - -ftpsync is a tool for mirroring (uploading) data to FTP servers written in Python. -It is built upon functionality in Stefan Schwarzer's ftputil package. -It has been tested with Python 2.3 and 2.4 and ftputil 2.1 under Linux. - -Usage: ftpsync.py [options] host source-dir target-dir -Known options: --exclude=, - --include=, - --exclude-from=, - --include-from=, - --delete, - --delete-excluded, - --dry-run, - --verbose, --quiet, --debug, - --trace=, - --cache-expire=, - --cache-size= - -Most options are equivalent to rsync(1)'s respective options. - -Features: - * include/exclude logic like rsync(1). - (Note: there is a script called rsync.py in the Python package index. - I have tested it and found it did not mimic rsync's logic correctly). - * Caching of FTP directory contents (simple FTPHost._dir() caching, - but speed up can be quite big) - * Deals with case-insensitive FTP server - -TODO: * download script - * proper packaging - * ... - -Files in this directory: - - ftpsync.py: main script - - caching_ftp.py: CachingFTPHost object, derived from ftputil - casepath.py: case-insensitive 'path' and 'stat' objects, derived from ftputil - - sync.py: abstract synchronizing logic - rsyncmatch.py: rsync-style globbing and include/exclude patterns - casestr.py: case-insensitive string class - simplecache.py: a very simplistic cache implementation - loggingclass.py: a small commodity layer above 'logging' Index: andbox/ftpsync-0.1/caching_ftp.py =================================================================== --- sandbox/ftpsync-0.1/caching_ftp.py (revision 607:f2ab969f29b5) +++ (revision ) @@ -1,127 +1,0 @@ -from ftputil import FTPHost -from ftputil.ftp_error import PermanentError, InternalError -from loggingclass import LoggingClass -from simplecache import Cache -from casepath import CaseInsPath, CaseInsStat - -class CachingFTPHost(FTPHost, LoggingClass): - - """ - This class is like ftputil.FTPHost, except that - the working directory and directory contents are cached. - This may speed up FTP operations significantly, especially - when traversing trees and looking for stat() like information. - - However, cached information may be wrong in some cases. - It is recommended to call host.invalidate_dir() after - closing the ftp_file object associated with . - - Constructor keywords "expire" and "size" are the same as for - simplecache.Cache. - - Besides, this class adds a method check_case_insensitive() to cope - with FTP servers that don't distinguish file names by case. - - """ - - def __init__(self, *args, **kwargs): - - kw = {} - if "expire" in kwargs: - kw["expire"] = kwargs["expire"] - del kwargs["expire"] - if "size" in kwargs: - kw["size"] = kwargs["size"] - del kwargs["size"] - self.cache = Cache(**kw) - - FTPHost.__init__(self, *args, **kwargs) - self.CWD = None - self.setcwd() - - def check_case_insensitive(self): - """ - Check for server case-insensivity and - This function must be called with an established connection and - with write permissions (like synchronize_times). - - """ - helper_name = "__CachingFtpHost_Helper__" - try: - self.mkdir(helper_name) - self.lstat(helper_name) # exception if mkdir failed - try: - # if the server is case-insensitive, this will fail - file = self.mkdir(helper_name.lower()) - except PermanentError: - self.logger.warning("Server is case-insensitive") - self.path = CaseInsPath(self) - self._stat = CaseInsStat(self) - self.cache.invalidate_all() - else: - self.logger.info("Server is case-sensitive") - finally: - self.rmdir(helper_name) - - def getcwd(self): - """ - Return cached working directory. - """ - return self.CWD - - def setcwd(self): - """ - Update cached working directory. - """ - self.CWD = self.path.normpath(FTPHost.getcwd(self)) - self.logger.info("New cwd: %s" % self.CWD) - - def chdir(self, path): - FTPHost.chdir(self, path) - self.setcwd() - - def _dir(self, path): - - path = self.path.normcase(path) - try: - lines = self.cache[path] - except KeyError: - self.logger.debug("cache miss: %s" % path) - lines = FTPHost._dir(self, path) - self.cache[path] = lines - else: - self.logger.debug("cache hit: %s" % path) - return lines - - def _invalidate_dir(self, path): - self.logger.debug("invalidating cache for %s" % path) - self.cache.invalidate( - self.path.normcase( - self.path.dirname(self.path.abspath(path)))) - - def file(self, path, mode='r'): - path = self.path.abspath(path) - ret = FTPHost.file(self, path, mode) - if 'w' in mode: - self._invalidate_dir(path) - return ret - - def mkdir(self, path, mode=None): - path = self.path.abspath(path) - FTPHost.mkdir(self, path, mode) - self._invalidate_dir(path) - - def rmdir(self, path): - FTPHost.rmdir(self, path) - self.cache.invalidate(self.path.normcase( - self.path.abspath(path))) - self._invalidate_dir(path) - - def remove(self, path): - FTPHost.remove(self, path) - self._invalidate_dir(path) - - def rename(self, source, target): - FTPHost.rename(self, source, target) - self._invalidate_dir(source) - self._invalidate_dir(target) Index: andbox/ftpsync-0.1/casepath.py =================================================================== --- sandbox/ftpsync-0.1/casepath.py (revision 607:f2ab969f29b5) +++ (revision ) @@ -1,113 +1,0 @@ -import posixpath -from ftputil import ftp_path, ftp_stat -from casestr import CaseInsStr - -class BasePath(ftp_path._Path): - """ - A reimplementation of ftputil.ftp_path._Path which is better suited - as base class than the original _Path. - """ - # ftputil.ftp_path._Path can't be inherited well because of the - # direct posixpath assignments in ftputil.ftp_path._Path.__init__(), - # which subclasses can't overload. - - # Here, dirname(), basename(), etc. are defined as methods which - # can be overloaded in derived classes. - - # This class inherits _Path in order to avoid duplicating - # code. Note that _Path.__init__() isn't called. - - def __init__(self, host, pp=posixpath): - self.pp = pp - self._host = host - - def dirname(self, *args): - return self.pp.dirname(*args) - - def basename(self, *args): - return self.pp.basename(*args) - - def isabs(self, *args): - return self.pp.isabs(*args) - - def commonprefix(self, *args): - return self.pp.commonprefix(*args) - - def join(self, *args): - return self.pp.join(*args) - - def split(self, *args): - return self.pp.split(*args) - - def splitdrive(self, *args): - return self.pp.splitdrive(*args) - - def splitext(self, *args): - return self.pp.splitext(*args) - - def normcase(self, *args): - return self.pp.normcase(*args) - - def normpath(self, *args): - return self.pp.normpath(*args) - - -class CaseInsPath(BasePath): - """ - A "path" implementation that treats paths in a case-insensitive manner. - The only difference to BasePath (and _Path) is that all - returned strings are 'CaseInsStr' objects. - """ - - def dirname(self, path): - return CaseInsStr(BasePath.dirname(self, path)) - - def basename(self, path): - return CaseInsStr(BasePath.basename(self, path)) - - def abspath(self, path): - return CaseInsStr(BasePath.abspath(self, path)) - - def normpath(self, path): - return CaseInsStr(BasePath.normpath(self, path)) - - def normcase(self, path): - return CaseInsStr(path.lower()) - - def join(self, *args): - return CaseInsStr(BasePath.join(self, *args)) - - def split(self, *args): - return [CaseInsStr(x) - for x in BasePath.split(self, *args)] - - def splitext(self, *args): - return [CaseInsStr(x) - for x in BasePath.splitext(self, *args)] - - def splitdrive(self, *args): - return [CaseInsStr(x) - for x in BasePath.splitdrive(self, *args)] - - -class CaseInsStat(ftp_stat._Stat): - """ - A class derived from _Stat that treats file names in a case insensitive - manner. - - E.g. "Spam" will be found and stat'd in a directory listing "spam, eggs". - """ - - def _stat_candidates(self, lines, wanted_name): - """Return candidate lines for further analysis.""" - ret = [line - for line in lines - if CaseInsStr(line).find(wanted_name) != -1] - return ret - - def _real_lstat(self, path, _exception_for_missing_path=True): - - path = CaseInsStr(path) - ret = ftp_stat._Stat._real_lstat(self, path, - _exception_for_missing_path) - return ret Index: andbox/ftpsync-0.1/casestr.py =================================================================== --- sandbox/ftpsync-0.1/casestr.py (revision 607:f2ab969f29b5) +++ (revision ) @@ -1,292 +1,0 @@ -import sys -from types import IntType, StringType - -class CaseInsStr(str): - """ - A reimplementation of the standard string class 'str' which - behaves in a case insensitive manner. - - See the python library documentation ("String methods") for - documentation of the methods. - - The case-insensitivity is greedy, i.e. operations between 'str' - and 'CaseInsStr' objects return 'CaseInsStr' objects. - All methods inherited from 'str' which would normally return - 'str' objects return 'CaseInsStr', including lower() and upper(). - - Use the str() method to convert to a normal, case-sensitive string. - - The string itself is not converted to lower or upper case. -""" - - def cast(self, str): - """ - Cast a "str" object into a "CaseInsStr" object. - """ - return CaseInsStr(str) - - def str(self): - """ - Convert to case-sensitive 'str' object. - """ - return self.__str__() - - def _lower(self): - return str.lower(self) - - def __cmp__(self, other): - """ - "CaseInsStr" objects can be compared with each other or with strings. - Objects are compared case-insensitively. - ->>> Spam = CaseInsStr("Spam") ->>> print "egg" < Spam, Spam >= "egg", "egg" < Spam.str(), Spam == "spam" -True True False True ->>> phrase = [CaseInsStr("Eric"), "the", "half", "a", "Bee"] ->>> phrase.sort() ->>> print phrase -['Bee', 'a', 'Eric', 'half', 'the'] - """ - ret = cmp(self._lower() , other.lower()) - return ret - - def __ne__(self, other): - return self.__cmp__(other) != 0 - - def __eq__(self, other): - return self.__cmp__(other) == 0 - - def __lt__(self, other): - return self.__cmp__(other) == -1 - - def __le__(self, other): - return self.__cmp__(other) != 1 - - def __gt__(self, other): - return self.__cmp__(other) == 1 - - def __ge__(self, other): - return self.__cmp__(other) != -1 - - def __getitem__(self, n): - return CaseInsStr(str.__getitem__(self, n)) - - def __getslice__(self, i, j): - return CaseInsStr(str.__getslice__(self, i, j)) - - def __add__(self, other): - """ ->>> # concatenation ->>> print "The "+CaseInsStr("Lovely ")+"Spam" == "the LOVELY spam" -True -""" - return CaseInsStr(str.__add__(self, other)) - - def __radd__(self, other): - return CaseInsStr(str.__add__(other, self)) - - def __mul__(self, n): - """ ->>> print 4 * CaseInsStr("Spam!") + CaseInsStr("Egg!") * 3 -Spam!Spam!Spam!Spam!Egg!Egg!Egg! -""" - return CaseInsStr(str.__mul__(self, n)) - - def __rmul__(self, n): - return CaseInsStr(str.__rmul__(self, n)) - - def center(self, width): - return CaseInsStr(str.center(self, width)) - - def count(self, sub, *args, **kwargs): - """ ->>> print CaseInsStr(4*"SPAM!").count("spam") -4 -""" - return self._lower().count(sub.lower(), *args, **kwargs) - - def find(self, other): - """ ->>> Love = CaseInsStr("The Lovely Spam") ->>> print Love.find("spam"), Love.rfind("love"), Love.index("ELY") -11 4 7 -""" - return self._lower().find(other.lower()) - - def index(self, other): - return self._lower().index(other.lower()) - - def join(self, seq): - """ ->>> print CaseInsStr("!").join(["spam", "Spam", "SPAM"]) -spam!Spam!SPAM ->>> print CaseInsStr("!").join(["spam", "Spam", "SPAM"]).find("SPAM") -0 -""" - return CaseInsStr(str.join(self, seq)) - - def ljust(self, width): - return CaseInsStr(str.ljust(self, width)) - - def replace(self, old, new, count=None): - """ ->>> # replace ->>> print CaseInsStr(4*"EGG!").replace("egg", "Spam", 3) -Spam!Spam!Spam!EGG! -""" - if count is not None and (type(count) != IntType or count < 0): - raise ValueError, count - old = old.lower() - lwr = self._lower() - n = 0 - idx = 0 - ret = "" - - while True: - i = lwr[idx:].find(old) - if i == -1 or (count is not None and n >= count): - break - ret = ret + str.__getslice__(self, idx, idx+i) + new - n = n + 1 - idx = idx + i + len(old) - - ret = ret + str.__getslice__(self, idx, sys.maxint) - return CaseInsStr(ret) - - def startswith(self, other): - return self._lower().startswith(other.lower()) - - def endswith(self, other): - return self._lower().endswith(other.lower()) - - def rfind(self, other): - return self._lower().rfind(other.lower()) - - def rindex(self, other): - return self._lower().rindex(other.lower()) - - def rjust(self, width): - return CaseInsStr(str.rjust(self, width)) - - def split(self, sep=None, maxsplit=0): - """ ->>> print CaseInsStr("Fiddle de dum, fiddle de dee").split("DE", 2) -['Fiddle ', ' dum, fiddle ', ' dee'] -""" - if sep is None: - return self.__str__().split(sep, maxsplit) - - if type(maxsplit) != IntType: - raise TypeError, maxsplit - if maxsplit < 0: - raise ValueError, maxsplit - - ret = [] - last = 0 - while True: - i = self[last:].find(sep) - if i == -1 or (maxsplit > 0 and len(ret) == maxsplit): - break - ret = ret + [self[last:last+i]] - last = last + i + len(sep) - - ret = ret + [self[last:]] - return ret - - def rsplit(self, sep=None, maxsplit=0): - """ ->>> print CaseInsStr("spam and eggs AND bees And knights").rsplit("and", 2) -['spam and eggs ', ' bees ', ' knights'] -""" - if sep is None: - return self.__str__().rsplit(sep, maxsplit) - - if type(maxsplit) != IntType: - raise TypeError, maxsplit - if maxsplit < 0: - raise ValueError, maxsplit - - ret = [] - last = sys.maxint - while True: - i = self[:last].rfind(sep) - if i == -1 or (maxsplit > 0 and len(ret) == maxsplit): - break - ret = [self[i+len(sep):last]] + ret - last = i - - ret = [self[:last]] + ret - return ret - - def splitlines(self, keepends=None): - return [CaseInsStr(x) - for x in str.splitlines(self, keepends)] - - def _stripchars(self, ch): - if type(ch) is not StringType: - raise TypeError, ch - s = "" - for x in ch: - if x.isalpha(): - u = x.upper() - l = x.lower() - if l != u: - s = s + l + u - continue - s = s + x - return s - - def strip(self, *chars): - """ - Stripped characters are case-insensitive: ->>> print CaseInsStr(" Spam and eggs ").strip("s ") -pam and egg ->>> print "'%s'" % CaseInsStr(" a ").lstrip() -'a ' -""" - if chars is () or chars[0] is None: - return CaseInsStr(str.strip(self)) - else: - return CaseInsStr(str.strip(self, self._stripchars(chars[0]))) - - def lstrip(self, *chars): - if chars is () or chars[0] is None: - return CaseInsStr(str.lstrip(self)) - else: - return CaseInsStr(str.lstrip(self, self._stripchars(chars[0]))) - - def rstrip(self, *chars): - if chars is () or chars[0] is None: - return CaseInsStr(str.rstrip(self)) - else: - return CaseInsStr(str.rstrip(self, self._stripchars(chars[0]))) - - def swapcase(self): - return CaseInsStr(str.swapcase(self)) - - def title(self): - return CaseInsStr(str.title(self)) - - def translate(self): - raise NotImplementedError, "translate() method not implemented" - - def lower(self): - """ ->>> Spam = CaseInsStr("Spam") ->>> print Spam.lower(), Spam.upper(), Spam.lower() == "SPAM", Spam.upper() == "spam" -spam SPAM True True -""" - return CaseInsStr(self._lower()) - - def upper(self): - return CaseInsStr(self.__str__().upper()) - - def zfill(self, width): - return CaseInsStr(str.zfill(self, width)) - -def _test(): - import doctest, casestr - doctest.testmod(casestr) - -if __name__ == "__main__": - _test() Index: andbox/ftpsync-0.1/ftpsync.py =================================================================== --- sandbox/ftpsync-0.1/ftpsync.py (revision 607:f2ab969f29b5) +++ (revision ) @@ -1,235 +1,0 @@ -import getopt -import netrc # for password retrieval from .netrc -import os -import sys -import termios # for getpass -import time -import traceback - -import loggingclass - -from caching_ftp import CachingFTPHost -from rsyncmatch import GlobChain -from sync import Synchronizer, RsyncSynchronizer - -# Copied from Python library manual (termios) -def getpass(prompt = "Password: "): - fd = sys.stdin.fileno() - old = termios.tcgetattr(fd) - new = termios.tcgetattr(fd) - new[3] = new[3] & ~termios.ECHO # lflags - try: - termios.tcsetattr(fd, termios.TCSADRAIN, new) - passwd = raw_input(prompt) - finally: - termios.tcsetattr(fd, termios.TCSADRAIN, old) - return passwd - - -def login_data(host): - """ - Derive login data for different FTP host formats: - - "hostname": check .netrc file, try anonymous otherwise - "user:pass@hostname": parse - "user@hostname": parse and ask for password interactively - """ - - user = "" - acct = "" - passwd = "" - - at = host.find("@") - if (at == -1): - try: - nrc = netrc.netrc() - (user, acct, passwd) = \ - nrc.authenticators(host) - except (IOError, TypeError): # no netrc file or no entry in netrc - pass - else: - user = host[:at] - host = host[(at+1):] - col = user.find(":") - if (col == -1): - passwd = getpass("password for ftp://%s%s: " % (user, host)) - else: - passwd = user[(col+1):] - user = user[:col] - - if (user == ""): - user = "anonymous" - - return (host, user, passwd, acct) - -def init_ftp(host, dir, **kw): - """ - Set up an FTP session for synchronizing (upload). - We must have write permissions in this case. - """ - data = login_data(host) - ftp = CachingFTPHost(*data, **kw) - ftp.chdir(dir) - ftp.synchronize_times() - ftp.check_case_insensitive() - return ftp - - -def start_logging(level, logfile): - """ - Initialize a useful logging environment with logging to stderr, - slightly more verbose to a log file, and suitable log levels. - """ - loggingclass.init_logging(level) - - if level == loggingclass.DEBUG: -# loggingclass.getLogger().setLevel(loggingclass.DEBUG) - loggingclass.set_default_level(loggingclass.INFO) - if logfile != "": - loggingclass.init_logfile(logfile, level=loggingclass.DEBUG) - loggingclass.set_class_level(RsyncSynchronizer, loggingclass.DEBUG) - loggingclass.set_class_level(GlobChain, loggingclass.DEBUG) - loggingclass.set_class_level(CachingFTPHost, loggingclass.INFO) - else: - if logfile != "": - loggingclass.init_logfile(logfile, level=loggingclass.INFO) - loggingclass.set_class_level(RsyncSynchronizer, loggingclass.INFO) - loggingclass.set_class_level(CachingFTPHost, loggingclass.INFO) - -def print_err(): - err = sys.exc_info() - printit = True - try: - if parm.level != loggingclass.DEBUG: - printit = False - except: - pass - if printit: - traceback.print_tb(err[2]) - sys.stderr.write("%s: %s\n" % (err[0], err[1])) - -class _Params: - """ - Class representing options and arguments for do_sync(). - """ - - class UsageError(Exception): - pass - - known = (list(GlobChain().options()) - + ["delete", "delete-excluded", "dry-run", - "verbose", "quiet", "debug", "trace=", - "cache-expire=", "cache-size="]) - - def usage(self): - sys.stderr.write("""\ -Usage: %s [options] host source-dir target-dir -Known options: %s -""" % (sys.argv[0], ", ".join(["--" + x for x in self.known]))) - raise self.UsageError() - - def _setlevel(self, level, option=True): - if self.level == -1: - self.level = level - elif option: - sys.stderr.write("Only one of --quiet, --debug, --verbose may be specified\n") - self.usage() - - def __init__(self): - self.level = -1 - self.dry_run = False - self.delete = False - self.delete_excluded = False - self.logfile = "" - self.expire = 300 - self.size = 2000 - - try: - (opts, args) = getopt.gnu_getopt(sys.argv[1:], "", self.known) - except getopt.GetoptError: - print_err() - self.usage() - - for (o, v) in opts: - if o == "--delete": - self.delete = True - elif o == "--delete-excluded": - self.delete_excluded = True - elif o == "--dry-run": - self.dry_run = True - elif o == "--verbose": - self._setlevel(loggingclass.INFO) - elif o == "--debug": - self._setlevel(loggingclass.DEBUG) - elif o == "--quiet": - self._setlevel(loggingclass.WARNING) - elif o == "--trace": - self.logfile=v - elif o == "--cache-expire": - self.expire = int(v) - elif o == "--cache-size": - self.size = int(v) - - self._setlevel(loggingclass.NOTICE, option=False) - if len(args) != 3: - self.usage() - - (self.host, self.source, self.target) = args - self.opts = opts - - -def _fix_source_n_target(parm): - # rsync-like semantics for trailing slash in source dir - if not os.path.isdir(parm.source): - raise ValueError, "%s is not a directrory" % parm.source - - if parm.source.endswith(os.sep): - parm.source = parm.source.rstrip(os.sep) - else: - parm.target = parm.ftp.path.join(parm.target, - os.path.basename(parm.source)) - if not parm.ftp.path.exists(parm.target): - parm.ftp.mkdir(parm.target) - -def log_checkpoint(msg): - loggingclass.getLogger().info( - "%s %s %s" % (__file__, msg, - time.strftime("%Y-%m-%d, %H:%M", time.localtime()))) - -def do_sync(parm): - - start_logging(parm.level, parm.logfile) - log_checkpoint("starting at") - - if parm.host == "localhost": - parm.ftp = os - else: - parm.ftp = init_ftp(parm.host, parm.target, - expire=parm.expire, size=parm.size) - - _fix_source_n_target(parm) - - sync = RsyncSynchronizer(os, parm.ftp, parm.source, parm.target, - delete = parm.delete, - dry_run = parm.dry_run, - delete_excluded = parm.delete_excluded) - - sync.globchain.getopt(parm.opts) - sync.sync("") - - log_checkpoint("finished at") - -if __name__ == "__main__": - try: - parm = _Params() - do_sync(parm) - except KeyboardInterrupt: - sys.stderr.write("Interrupted.\n") - sys.exit(0) - except _Params.UsageError: - sys.exit(129) - except: - print_err() - sys.exit(130) - else: - sys.exit(0) Index: andbox/ftpsync-0.1/loggingclass.py =================================================================== --- sandbox/ftpsync-0.1/loggingclass.py (revision 607:f2ab969f29b5) +++ (revision ) @@ -1,118 +1,0 @@ -from logging import * -import sys - -NOTICE = (INFO+WARNING)/2 -_default_level = NOTICE -_default_format = "%(filename)s:%(name)s[%(lineno)d]: %(message)s" - -def set_default_level(level): - """ - Set the default log level for classes for which set_class_level() - or instance.set_log_level() was never called. - Default: NOTICE. - Call early - only affects class loggers created after the call. - """ - global _default_level - _default_level = level - -def _class_logger(cls): - nm ="_" + cls.__name__ + "__logger" - try: - lg = getattr(cls, nm) - except AttributeError: - lg = getLogger(cls.__name__) - lg.setLevel(_default_level) - setattr(cls, nm, lg) - return lg - -def set_class_level(cls, level): - """ - Set the log level for this class. - level: one of the levels defined in the logging module. - Side effects: Sets the log level for _any object_ of this class. - """ - lg = _class_logger(cls) - lg.setLevel(level) - -class LoggingClass: - - """ - A base class for classes using for easy use of the "logging" module. - - Instances of derived classes have a "logger" attribute - that represents a class-specific logging.Logger object. - - The "name" of the class-specific logger will be the class name. - Log levels etc. can be set on a class-specific basis. - - CAUTION: The "logger" attribute is resolved through __getattr__(). - The "__logger" attribute should work independently of __getattr__(). - - Usage (doctest): - ->>> class LogTest(LoggingClass): -... def info(self): -... self.logger.info("This is an info.") -... def error(self): -... self.logger.error("This is an error!") -... ->>> init_logging(level=INFO, -... format="%(name)s[%(lineno)d]: %(message)s", stream=sys.stdout) ->>> logtest = LogTest() ->>> logtest.info() ->>> logtest.error() -LogTest[5]: This is an error! ->>> LogTest().set_log_level(INFO) ->>> logtest.info() -LogTest[3]: This is an info. - - """ - - # We can't simply define a "logger" attribute because we - # want class-specific loggers. - - # The attribute name is constructed such that self.__logger - # will work in derived classes. - - def __getattr__(self, attr): - """ - Resolves the "logger" attribute. - Call this when redefining __getattr__() in derived classes! - """ - if attr == "logger": - return _class_logger(self.__class__) - raise AttributeError, ("'LoggingClass' object has no attribute '%s'" - % attr) - - def set_log_level(self, level): - """ - Set the log level for this class. - level: one of the levels defined in the logging module. - Sets the log level for _any object_ of this class. - """ - return set_class_level(self.__class__, level) - - -def init_logging(level=NOTICE, format=_default_format, stream=sys.stderr): - """Initialize a basic logging setup.""" - - handler = StreamHandler(stream) - handler.setFormatter(Formatter(format)) - handler.setLevel(level) - - getLogger().addHandler(handler) - -def init_logfile(file, level=INFO, format=_default_format): - - handler = FileHandler(file) - handler.setFormatter(Formatter(format)) - handler.setLevel(level) - - getLogger().addHandler(handler) - -def _test(): - import doctest, loggingclass - doctest.testmod(loggingclass) - -if __name__ == "__main__": - _test() Index: andbox/ftpsync-0.1/rsyncmatch.py =================================================================== --- sandbox/ftpsync-0.1/rsyncmatch.py (revision 607:f2ab969f29b5) +++ (revision ) @@ -1,437 +1,0 @@ -import re -import os -import sys -import loggingclass - -INCLUDE = "+" -EXCLUDE = "-" -DONE = "." - -class RsyncGlob(loggingclass.LoggingClass): - """ - A class that imitates rsync(1)'s way of include/exclude patterns. - Similar to glob()/fnmatch(), but "*" doesn't match "/" - "**" does. - See man rsync(1) for the exclude/include logic. - The GlobChain class creates filter chains like rsync's. - - RsyncGlob(pattern), where follows the rules in the rsync(1) - man page. In particular, "/XYZ" matches only at the "root", and "XYZ/" - matches only directories. - - NOTE: This class uses Unix file name conventions. - It will be pretty simple to implement it for DOS/Windows though, - if someone volunteers. - - The following doctest example shows how the globbing works. - Note that leading "/" are stripped for files. - ->>> globs=(RsyncGlob("s*m"),RsyncGlob("s**m"), -... RsyncGlob("s\*m"),RsyncGlob("/s*m"), -... RsyncGlob("/s**m"),RsyncGlob("s\**m")) ->>> files=("spam","s/p/a/m","egg/spam","egg/sp/am","s*m","s*am") ->>> def outh(): -... s="%10.10s" %"" -... for g in globs: -... s=s+"%10.10s"%g.glob -... return s ->>> def outf(f): -... s="%10.10s" %f -... for g in globs: -... s=s+"%10.10s"%g.match(f) -... return s ->>> def out(): -... print outh() -... for f in files: -... print outf(f) ->>> out() - s*m s**m s\*m /s*m /s**m s\**m - spam True True False True True False - s/p/a/m False True False False True False - egg/spam True True False False False False - egg/sp/am False True False False False False - s*m True True True True True True - s*am True True False True True True - """ - - # This is applied before escaping re metachars - __bksl_re = re.compile(r'(\\.)') - - # These are applied after escaping re metachars - __star2_re = re.compile(r'(\\\*\\\*)') # "**" - __star_re = re.compile(r'(\\\*)') # "*" - __quest_re = re.compile(r'(\\\?)') # "?" - __slash_re = re.compile(r'/') # "/" - - def __handle_stars(self, s): - parts = self.__star2_re.split(s) - - # side effect: patterns containing "**" match complete path - self.path_match = self.path_match or (len(parts) > 1) - - # Odd elements are '**' now. - # Even elements must be checked for '*' and '?' - res = "" - i = 0 - while i < len(parts): - if parts[i] != "": - tmp = self.__star_re.sub("[^/]*", parts[i]) - tmp = self.__quest_re.sub("[^/]", tmp) - res = res + tmp - if i < len(parts) - 1: - res = res + ".*" - i = i + 2 - return res - - - def __init__(self, pat=""): - - self.type = None - - # patterns starting with +/-. - if len(pat) > 1: - if pat[:2] == "+ ": - self.type = INCLUDE - pat = pat[2:] - elif pat[:2] == "- ": - self.type = EXCLUDE - pat = pat[2:] - - # patterns ending with "/" match only directories - if len(pat) > 0 and pat.endswith("/"): - self.dir_match = True - pat = pat[:-1] - else: - self.dir_match = False - - self.glob = pat - - # patterns containing "/" match entire path, - self.path_match = (pat.find("/") != -1) - - # patterns starting with "/" match only at root - if len(pat) > 0 and pat[0] == "/": - pat = pat[1:] - top_match = True - else: - top_match = False - - # We transform the glob pattern into a regexp pattern now. - # First, handle all characters escaped with backslashes. - parts = self.__bksl_re.split(pat) - - i = 0 - self.pat = "" - - # Odd elements of parts are an escaped chars now. - # Need to look for glob patterns in even elements. - while i < len(parts): - if parts[i] != "": - # escape any remaining regexp metacharacters like "." - s = re.escape(parts[i]) - # sort out "**" and "*" - s = self.__handle_stars(s) - self.pat = self.pat + s - # Add back the escaped chars - if (i < len(parts) - 1): - self.pat = self.pat + parts[i+1] - i = i+2 - - self.pat = self.pat + "$" - if top_match: - self.pat = "^" + self.pat - # Special case: '**/' matches empty string - elif self.pat[:4] == r".*\/": - self.pat = "(.*/|)" + self.pat[4:] - - self.logger.debug("regexp: %s -> (%s)" % (self, self.pat)) - self.re = re.compile(self.pat) - - - def __str__(self): - s = self.glob - if self.dir_match: s = s + "/" - - if self.type: - t = self.type - else: - t = " " - if self.path_match: - p="p" - else: - p=" " - return "(%s)[%s%s]" % (s, t, p) - - - def match(self, filename): - - if len(filename) > 0 and filename.endswith("/"): - filename = filename [:-1] - elif self.dir_match: - return False - - if self.path_match: - ret = self.re.search(filename) is not None - else: - ret = self.re.match(os.path.basename(filename)) is not None - - if ret: - self.logger.debug("%s matches %s" % (filename, self)) - return ret - - -class GlobChain(loggingclass.LoggingClass): - """ - A class that represents a chain of RsyncGlob filter rules. - Filter rules are applied in order. The recurse() function can - be used to filter directories recursively. - - doctest example: - ->>> loggingclass.init_logging(level=loggingclass.DEBUG, -... format="%(name)s[%(lineno)d]: %(message)s", -... stream=sys.stdout) ->>> ->>> ch = GlobChain() ->>> ch.set_log_level(loggingclass.DEBUG) ->>> ->>> ch.exclude("+ spam/", "- /*/", "+ egg/", "- */", "+ \*", "- *") -GlobChain[251]: added rule: (spam/)[+ ] -GlobChain[251]: added rule: (/*/)[-p] -GlobChain[251]: added rule: (egg/)[+ ] -GlobChain[251]: added rule: (*/)[- ] -GlobChain[251]: added rule: (\*)[+ ] -GlobChain[251]: added rule: (*)[- ] ->>> for x in ("spam", "spam/", "egg", -... "egg/", "*", "spam/egg", -... "spam/egg/", "spam/*/egg", "spam/egg/*"): -... xx = ch.match(x) -GlobChain[279]: exclude spam -GlobChain[279]: include spam/ -GlobChain[279]: exclude egg -GlobChain[279]: exclude egg/ -GlobChain[279]: include * -GlobChain[279]: exclude spam/egg -GlobChain[279]: include spam/egg/ -GlobChain[279]: exclude spam/*/egg -GlobChain[279]: include spam/egg/* - """ - - __end_re = re.compile(r"/+$") - - def __init__(self): - - self._lst = [] - - def _in_ex(self, x): - if x == INCLUDE: - return "include" - elif x == EXCLUDE: - return "exclude" - else: - return None - - def add(self, type, *args): - """ - add(type, *args): add (a) new INCLUDE/EXCLUDE pattern rule(s). - is either INCLUDE or EXCLUDE, or the patterns must - start with "+" or "-". - +/- start character has precedence over . - """ - for x in args: - # "!" resets the rule chain (why?) - if (x == "!"): - l = [] - else: - glb = RsyncGlob(x) - if (glb.type == None): - if (type == None): - raise ValueError, "filter type is undefined for %s" % glb - else: - glb.type = type - self.logger.info("added rule: %s" % glb) - self._lst.append(glb) - - def exclude(self, *args): - """ - exclude(*args): add (a) new EXCLUDE pattern rule(s) - """ - self.add(EXCLUDE, *args) - - def include(self, *args): - """ - include(*args): add (a) new INCLUDE pattern rule(s) - """ - self.add(INCLUDE, *args) - - def match(self, path): - """match(path): returns the result of the current filter chain for path. - The rule chain is traversed until the first rule matches. - If path is a directory, it should end in "/". - """ - - # Default is always INCLUDE - ret = INCLUDE - for glb in self._lst: - if glb.match(path): - ret = glb.type - break - - self.logger.debug("%s %s" % (self._in_ex(ret), path)) - return ret - - def _recurse(self, top, dir, collector, *args): - - for f in os.listdir(os.path.join(top, dir)): - rel = os.path.join(dir, f) - isdir = os.path.isdir(os.path.join(top, rel)) - - pat = rel - if (isdir): - pat = pat + "/" - - c = self.match(pat) - - collector(c, pat, *args) - if c == EXCLUDE: - continue - - if isdir: - self._recurse(top, rel, collector, *args) - - def collect(self, c, x, *args): - """ - Default collector function to use with recurse(). - """ - l = args[0] - if c == INCLUDE: - l.append(x) - elif c == DONE: - return l - - def recurse(self, dir, collector = None, *args): - """ - recurse(self, dir, collector = None, *args) - recursively descend . For each file or directory found, - is called with arguments (c, x, *args), where - is the result of the test chain (or DONE when finished), - is the current path, and <*args> is the rest of the arguments - of recurse(). - - Directories for which the chain evaluates to EXCLUDE are never entered. - - When c == DONE, should return its final result. This will - be the return value of recurse(). - - If isn't set, a default collector function is used that - returns a list of all files below for which c == INCLUDE. - """ - - if collector == None: - collector = self.collect - args = ([],) - - dir = self.__end_re.sub("", dir) - if not os.path.isdir(dir): - raise IOError, "%s is not a directory" % dir - - self._recurse(dir, "", collector, *args) - ret = collector(DONE, None, *args) - return ret - - def add_file(self, type, name): - """ - add_file(type, name): - Add a list of INCLUDE/EXCLUDE filter rules from a file. - Used to implement --exclude-from, --include-from. - """ - try: - if name == "-": - f = sys.stdin - else: - f = open(name, "r") - - for line in f.readlines(): - if line.endswith("\n"): - line = line[:-1] - self.add(type, line) - finally: - if name != "-": - f.close() - - _known_options = ("exclude=", "include=", - "exclude-from=", "include-from=") - - def options(self): - return self._known_options - - def getopt(self, options): - """ - getopt(options): parse getopt-style option pairs for filter rules. - Parses all options "--exclude=", "--include=", "--exclude-from=", - "--include-from=", leaving other options untouched. - See rsync(1) for the option semantics. - """ - hits = [] - for i in range(0, len(options)): - (name, val) = options[i] - hit = True - if name == "--exclude": - self.exclude(val) - elif name == "--include": - self.include(val) - elif name == "--exclude-from": - self.add_file(EXCLUDE, val) - elif name == "--include-from": - self.add_file(INCLUDE, val) - else: - hit = False - if hit: - hits.append(i) - - hits.reverse() - for i in hits: - del options[i] - - -def print_matches(): - """ - Usage: rsyncmatch.py [--debug] [filter rules...] directory ... - - Print list of files below that match the given rules. - Rules are specified with "--exclude=", "--include=", "--exclude-from=", - "--include-from=", see rsync(1) for rule semantics. - """ - - import getopt - import logging - - loggingclass.init_logging() - - - gl = GlobChain() - (options, args) = getopt.gnu_getopt(sys.argv[1:], "", - (gl._known_options) + ("debug",)) - gl.getopt(options) - for x in options: - if x[0] == "--debug": - GlobChain().set_log_level(loggingclass.DEBUG) - RsyncGlob().set_log_level(loggingclass.DEBUG) - - for dir in args: - ls = gl.recurse(dir) - - print "List of include files below %s:" % dir - for x in ls: - print " " + x - - -def _test(): - import doctest, rsyncmatch - doctest.testmod(rsyncmatch) - -if __name__ == "__main__": - if sys.argv[1] == "--test": - sys.argv = sys.argv[1:] - _test() - else: - print_matches() Index: andbox/ftpsync-0.1/simplecache.py =================================================================== --- sandbox/ftpsync-0.1/simplecache.py (revision 607:f2ab969f29b5) +++ (revision ) @@ -1,225 +1,0 @@ -import threading -import loggingclass -import time - -class CacheEntry: - """ - A class representing a cache entry. - x = CacheEntry() - """ - - def __init__(self, val): - self.val = val - self.stamp = time.time() - - def expired(self, period): - """ - Boolean: returns true if the entry is older than period (in sec). - """ - return time.time() - self.stamp > period - - def __cmp__(self, other): - """ - CacheEntry objects can be compared by age. - """ - return cmp(self.stamp, other.stamp) - - -class Cache(loggingclass.LoggingClass): - """ - A simple cache implementation - - Usage: c = Cache(expire=, size=, entryclass=CacheEntry) - : expiration time of entries in sec (default: 60) - : max number of cache entries (default: 1000) - : A class for the cache entries (default: CacheEntry) - - NOTE: EXPIRED ENTRIES WILL BE DELETED. - Do not use this class (exclusively) to store valuable data. - - Entries are assigned and retrieved through indexing: - cache[x] = val - try: - val = cache[x] - except KeyError: - print x, ": not in cache" - cache.invalidate(x) - - doctest example: - ->>> from time import sleep ->>> exp=2.0 ->>> cache = Cache(size=10, expire=exp) ->>> for x in range(0, 10): -... cache[x] = x*x ->>> print cache.contents() -[(0, 0), (1, 1), (2, 4), (3, 9), (4, 16), (5, 25), (6, 36), (7, 49), (8, 64), (9, 81)] ->>> for x in range(11, 20): -... cache[x] = x*x ->>> ->>> # size exceeded - old elements will be deleted ->>> print cache.contents() -[(11, 121), (12, 144), (13, 169), (14, 196), (15, 225), (16, 256), (17, 289), (18, 324), (19, 361)] ->>> sleep(exp/2.) ->>> for x in range(0, 5): -... cache[x] = x*x ->>> print cache.contents() -[(0, 0), (1, 1), (2, 4), (3, 9), (4, 16), (16, 256), (17, 289), (18, 324), (19, 361)] ->>> sleep(exp/2.+0.1) ->>> ->>> # elements 11 .. 20 will be expired ->>> print cache.contents() -[(0, 0), (1, 1), (2, 4), (3, 9), (4, 16)] - """ - - default_expire = 60 - default_size = 1000 - _to_shrink = 100 # No. of entries to delete in a _shrink() call - - def __init__(self, expire = default_expire, size = default_size, - entryclass = CacheEntry): - self.__cache = {} - self.__size = size - self.expire = expire - self.__lock = threading.Lock() - self._entryclass = entryclass - - def _lock(self): - self.__lock.acquire() - - def _unlock(self): - self.__lock.release() - - def _invalidate(self, key): - if self.__cache.has_key(key): - del self.__cache[key] - return True - return False - - def len(self): - """ - Return current number of cache entries. - """ - return len(self.__cache) - - def invalidate(self, key): - """ - Invalidate (delete) cache entry indexed by key - """ - self._lock() - try: - if self._invalidate(key): - self.logger.debug("element %s invalidated" % key) - finally: - self._unlock() - - def invalidate_all(self): - """ - Clear cache completetly - """ - self._lock() - try: - self.__cache.clear() - finally: - self._unlock() - self.logger.info("cache cleared") - - def invalidate_some(self, func): - """ - Invalidate all entries for which func(key,val) returns True - """ - self._lock() - try: - for x in self.__cache.keys(): - if func(x, self.__cache[x].val): - self._invalidate(x) - self.logger.debug("element %s invalidated" % x) - finally: - self._unlock() - - def _expired(self, entry): - return self.expire != 0 and entry.expired(self.expire) - - def __getitem__(self, key): - """ - Implements x = cache[key]. - Will raise KeyError if the cache entry is expired. - """ - ret = None - self._lock() - try: - entry = self.__cache[key] - if self._expired(entry): - self._gc() - raise KeyError - else: - ret = entry.val - finally: - self._unlock() - - return ret - - def __setitem__(self, key, val): - - """ - Implements cache[key] = y. - """ - self._lock() - try: - self._invalidate(key) - if self.len() >= self.__size: - self._shrink() - self.__cache[key] = self._entryclass(val) - finally: - self._unlock() - - def _shrink(self): - # Must be called with lock held - target = self.__size - min(self.__size/2, self._to_shrink) - self.logger.debug("_shrink: trying to reduce size from %d to %d" - % (self.len(), target)) - - self._gc() - - n = self.len() - target - if n <= 0: - return - - # sort entries by age and invalidate the n oldest - keys = self.__cache.keys() - keys.sort(lambda a, b, c=self.__cache: cmp(c[a], c[b])) - - self.logger.info("_shrink: invalidating %d entries" % n) - for key in keys[:n]: - self._invalidate(key) - - def _gc(self): - # Must be called with lock held - expired = [] - for key in self.__cache.keys(): - if self._expired(self.__cache[key]): - expired.append(key) - for key in expired: - self._invalidate(key) - self.logger.info("collecting garbage: %d items invalidated" - % len(expired)) - - def contents(self): - """ - returns a list of (key, val) tuples for all non-expired entries - """ - self._lock() - try: - self._gc() - ret = [(x, self.__cache[x].val) for x in self.__cache.keys()] - finally: - self._unlock() - return ret - - -def _test(): - import doctest, simplecache - doctest.testmod(simplecache) - -if __name__ == "__main__": - _test() Index: andbox/ftpsync-0.1/sync.py =================================================================== --- sandbox/ftpsync-0.1/sync.py (revision 607:f2ab969f29b5) +++ (revision ) @@ -1,388 +1,0 @@ -import os -import sys -from loggingclass import LoggingClass, NOTICE -from rsyncmatch import GlobChain, EXCLUDE - -class Synchronizer(LoggingClass): - """ - A class for synchronizing directories between two file systems. - - Usage example: - sync = Synchronizer(os, os, "/source", "/target") - sync.sync("subdir") - - This will synchronize "/source/subdir" to "/target/subdir". - """ - - class SyncAction: - """ - This "class" stores actions to be carried out. - """ - def __init__(self): - self.unl = [] # stuff to unlink - self.cpy = [] # stuff to copy - self.rmd = [] # stuff to rmdir - self.mkd = [] # stuff to mkdir - self.dsc = [] # dirs to descend into - - class FileSys: - """ - A helper class for Synchronizer. Another abstraction layer above - 'os' and other filesystem access (e.g. FTP). - - It inherits most attributes from it's '_io' element (typically 'os'). - """ - - def __init__(self, io, root): - self._io = io - self.root = root - - def open(self, *args): - """ - Open a file on the file system. - """ - if self._io == os: - return open(*args) - else: - return self._io.open(*args) - - def eq(self, x, y): - """ - Boolean: True if file names x and y are equal by this file - system's rules. This refers mainly to case-sensitiveness. - """ - ret = (self._io.path.normcase(x) == self._io.path.normcase(y)) - return ret - - def cmp(self, x, y): - """ - Compare file names by this file system's rules. - """ - ret = cmp(self._io.path.normcase(x), self._io.path.normcase(y)) - return ret - - def __getattr__(self, attr): - return getattr(self._io, attr) - - - def __init__(self, io_s, io_t, root_s, root_t, - mode = "b", blocksize = 65536, - delete=False, delete_excluded=False, - dry_run=False): - """ - io_s, io_t: "IO class" of the source and target, respectively. - typically 'os' or an ftputil.FTPHost - root_s, root_t: root directories for synchronization on source - and target, respectively. - mode: file open() mode (usually 'b') - blocksize: block size for copying (default: 64kB) - delete: whether to delete additional files on target (default: false) - delete_excluded: whether to delete files which were excluded, similar - to rsync's --delete-exluded option. See exclude() method. - dry_run: whether anything should actually be done on the target. - """ - - self.io_s = self.FileSys(io_s, root_s) - self.io_t = self.FileSys(io_t, root_t) - self.mode = mode - self.dry_run = dry_run - self.blocksize = blocksize - self.delete = delete - self.delete_excluded = delete_excluded - self.logger.info("options: delete=%s, delete-excluded=%s, dry-run=%s" - % (self.delete, self.delete_excluded, self.dry_run)) - return - - def _rm_rf(self, path): - err = False - for f in self.io_t.listdir(path): - absl = self.io_t.path.join(path, f) - if self.isdir(self.io_t, absl): - try: - self._rm_rf(absl) - except OSError: - self.logger.exception("rmdir %s" % absl) - err = sys.exc_info()[:2] - else: - self.logger.debug("delete %s" % absl) - try: - if not self.dry_run: - self.io_t.unlink(absl) - except OSError: - self.logger.exception("delete %s" % absl) - err = sys.exc_info()[:2] - self.logger.debug("rmdir %s" % path) - if not self.dry_run: - self.io_t.rmdir(path) - if err: - raise err[0], err[1] - - def rm_rf(self, path): - """ - Remove directory recursively. - """ - absl=self.io_t.path.abspath(path) - self._rm_rf(absl) - - def _pull(self, x, lst, eq): - """ - x: file name - lst: list of file names - eq: function to check file name equality - returns: true if file was matched - side effects: removes all matching entries from lst - """ - oldlen = len(lst) - i = 0 - while i < len(lst): - if eq(x, lst[i]): - found = True - del lst[i] - i = i + 1 - return (len(lst) < oldlen) - - def exclude(self, dir, name, isdir): - """ - (virtual): this implementation returns always False. - dir: parent directory - name: file name - isdir: True iff name represents a directory itself - returns: True if file is to be excluded. - """ - return False - - def need_copy(self, src, tgt): - """ - src, tgt: corresponding files on source and target - returns: a "reason string" if src needs to be copied to tgt. - the emtpy string otherwise. - This default implementation returns non-"" if the file - sizes differ ("size"), or if src is newer than tgt ("date"). - """ - ret = "" - stat_s = self.io_s.stat(src) - stat_t = self.io_t.stat(tgt) - if stat_s.st_size != stat_t.st_size: - ret = "size" - self.logger.debug("%s: sizes differ: %d %d" % - (tgt, stat_s.st_size, stat_t.st_size)) - elif (stat_s.st_mtime - stat_t.st_mtime > 0): - ret = "date" - self.logger.debug("%s: source is newer by %s s" % - (tgt, stat_s.st_mtime - stat_t.st_mtime)) - return ret - - def _make_pattern(self, path, isdir): - if isdir: - path = path + "/" - return path - - def isdir(self, io, path): - return io.path.isdir(path) and not io.path.islink(path) - - def copy(self, abs_s, abs_t): - try: - src = self.io_s.open(abs_s, "r" + self.mode) - tgt = self.io_t.open(abs_t, "w" + self.mode) - while True: - buffer = src.read(self.blocksize) - if not buffer: break - tgt.write(buffer) - except(IOError, OSError): - self.logger.exception("error copying to %s" % abs_t) - try: - self.io_s.unlink(abs_t) - except(IOError, OSError): - self.logger.exception("error unlinking %s" % abs_t) - pass - - try: - src.close() - tgt.close() - except: - pass - - def _unique(self, lst, eq): - """ - Remove duplicate entries in list lst, using equality relation eq. - """ - i = 0 - while i < len(lst): - j = i + 1 - while j < len(lst): - if eq(lst[i], lst[j]): - self.logger.warn("skipping %s (duplicate of %s)" - % (lst[j], lst[i])) - del lst[j] - else: - j = j + 1 - i = i + 1 - - def sync(self, path, _top=True): - """ - Main work horse of Synchorinzer class. - Synchronize directory 'path' between source and target. - - Called recursively. Call with _top = True initially. - """ - - # All action items are recorded in this "todo" list. - # Actions are only put into effect when the list is - # complete. - todo = self.SyncAction() - # 'reason' is a map that stores the reasons why we transfer - # files (regular files only). This is just informational. - reason = {} - - path_s = self.io_s.path.join(self.io_s.root, path) - path_t = self.io_t.path.join(self.io_t.root, path) - - if _top: - self.logger.info("sync starting: %s -> %s" % (path_s, path_t)) - else: - self.logger.debug("sync: %s -> %s" % (path_s, path_t)) - - lst_s = self.io_s.listdir(path_s) - try: - lst_t = self.io_t.listdir(path_t) - except OSError: - if self.dry_run: - lst_t = [] - else: - raise - - # in case io_s or io_t are case-insensitive, remove duplicate - # file names. - self._unique(lst_s, self.io_t.eq) - self._unique(lst_t, self.io_s.eq) - - for x in lst_s: - - abs_s = self.io_s.path.join(path_s, x) - isdir_s = self.isdir(self.io_s, abs_s) - - if not isdir_s and not self.io_s.path.isfile(abs_s): - self.logger.info("skipping non-file %s" % abs_s) - continue - - # This deletes x from lst_t. That enables us to simply - # iterate over lst_t later to find files to be deleted. - exists_t = self._pull(x, lst_t, self.io_t.eq) - abs_t = self.io_t.path.join(path_t, x) - if exists_t: - isdir_t = self.isdir(self.io_t, abs_t) - - if self.exclude(path, x, isdir_s): - self.logger.debug("exclude src %s/%s" % (path, x)) - if self.delete and self.delete_excluded and exists_t: - self.logger.log(NOTICE, "delete excluded %s/%s" % (path, x)) - if isdir_t: - todo.rmd.append(x) - else: - todo.unl.append(x) - continue - - # Here we know: src exists and is not excluded. - if isdir_s: - todo.dsc.append(x) - - if exists_t: - if isdir_s: - if not isdir_t: - todo.unl.append(x) - todo.mkd.append(x) - else: - if isdir_t: - todo.rmd.append(x) - todo.cpy.append(x) - reason[x] = "type" - else: - rsn = self.need_copy(abs_s, abs_t) - if rsn: - todo.unl.append(x) - todo.cpy.append(x) - reason[x] = "%s" % rsn - else: # not exists_t - if isdir_s: - todo.mkd.append(x) - else: - reason[x] = "new" - todo.cpy.append(x) - # for loop over lst_s ends - - if self.delete: - - # Anything now in lst_t didn't exist in src (see above) - for x in lst_t: - - abs_t = self.io_t.path.join(path_t, x) - isdir_t = self.isdir(self.io_t, abs_t) - - if self.exclude(path, x, isdir_t): - self.logger.debug("exclude tgt %s/%s" % (path, x)) - if not self.delete_excluded: - continue - - self.logger.info("delete %s/%s" % (path, x)) - if isdir_t: - todo.rmd.append(x) - else: - todo.unl.append(x) - - # From here on ACTIONS ARE CARRIED OUT - # First all remove actions, than mkdir and copy - for x in todo.rmd: - try: - self.logger.log(NOTICE, "rm -rf: %s/%s" % (path, x)) - self.rm_rf(self.io_t.path.join(path_t, x)) - except (OSError, IOError): - self.logger.exception("failed to rmdir %s" % x) - - for x in todo.unl: - try: - self.logger.log(NOTICE, "delete: %s/%s" % (path, x)) - if not self.dry_run: - self.io_t.unlink(self.io_t.path.join(path_t, x)) - except (OSError, IOError): - self.logger.exception("failed to unlink %s" % x) - - for x in todo.mkd: - try: - self.logger.log(NOTICE, "mkdir: %s/%s" % (path, x)) - if not self.dry_run: - self.io_t.mkdir(self.io_t.path.join(path_t, x)) - except (OSError, IOError): - self.logger.exception("failed to mkdir %s" % x) - self._pull(x, todo.dsc) - - for x in todo.cpy: - self.logger.log(NOTICE, "copy: %s/%s (reason: %s)" - % (path, x, reason[x])) - if not self.dry_run: - self.copy(self.io_s.path.join(path_s, x), - self.io_s.path.join(path_t, x)) - - # Finally, recurse. - for x in todo.dsc: - self.sync(self.io_s.path.join(path, x), False) - - if _top: - self.logger.info("sync finshed: %s -> %s" % (path_s, path_t)) - - -class RsyncSynchronizer(Synchronizer): - """ - Special Synchronzer class that uses rsyncmatch.GlobChain - for include/exclude logic. - """ - def __init__(self, *args, **kwargs): - Synchronizer.__init__(self, *args, **kwargs) - self.globchain = GlobChain() - - def exclude(self, dir, name, isdir): - path = self.io_s.path.join(dir, name) - if isdir: - path = path + "/" - - gl = self.globchain.match(path) - return gl == EXCLUDE Index: sandbox/upload_download_test.py =================================================================== --- sandbox/upload_download_test.py (revision 609:5a9c27304169) +++ sandbox/upload_download_test.py (revision 567:72ebc25b5c85) @@ -32,5 +32,5 @@ # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -# $Id: upload_download_test.py 633 2006-11-22 22:15:32Z schwa $ +# $Id: upload_download_test.py 570 2006-10-14 01:33:14Z schwa $ # Test script for ticket #13 (reported by Pete Schott) Index: setup.py =================================================================== --- setup.py (revision 827:7920a39a379c) +++ setup.py (revision 600:7d901252fc5e) @@ -1,5 +1,5 @@ #! /usr/bin/env python -# Copyright (C) 2003-2009, Stefan Schwarzer +# Copyright (C) 2003-2006, Stefan Schwarzer # All rights reserved. # @@ -38,10 +38,8 @@ """ -import os import sys from distutils import core from distutils import sysconfig -from distutils.command import install_lib as install_lib_module @@ -49,16 +47,5 @@ _package = "ftputil" _version = open("VERSION").read().strip() - - -# avoid byte-compiling `_test_with_statement.py` for Python < 2.5; see -# http://mail.python.org/pipermail/distutils-sig/2002-June/002894.html -class FtputilInstallLib(install_lib_module.install_lib): - def byte_compile(self, files): - if sys.version_info < (2, 5): - files = [f for f in files - if os.path.basename(f) != "_test_with_statement.py"] - # `super` doesn't work with classic classes - return install_lib_module.install_lib.byte_compile(self, files) - +_data_target = "%s/%s" % (sysconfig.get_python_lib(), _package) core.setup( @@ -68,8 +55,6 @@ packages=[_package], package_dir={_package: ""}, - data_files=[("doc", ["ftputil.txt", "ftputil.html", - "README.txt", "README.html"])], - cmdclass={'install_lib': FtputilInstallLib}, - + data_files=[(_data_target, ["ftputil.txt", "ftputil.html", + "README.txt", "README.html"])], # metadata author="Stefan Schwarzer", @@ -79,5 +64,5 @@ keywords="FTP, client, virtual file system", license="Open source (revised BSD license)", - platforms=["Pure Python (Python version >= 2.3)"], + platforms=["Pure Python (Python version >= 2.1)"], long_description="""\ ftputil is a high-level FTP client library for the Python programming @@ -91,5 +76,5 @@ (_name, _version), classifiers=[ - "Development Status :: 5 - Production/Stable", + "Development Status :: 6 - Mature", "Environment :: Other Environment", "Intended Audience :: Developers",