1 | # Copyright (C) 2006-2018, Stefan Schwarzer <sschwarzer@sschwarzer.net> |
---|
2 | # and ftputil contributors (see `doc/contributors.txt`) |
---|
3 | # See the file LICENSE for licensing terms. |
---|
4 | |
---|
5 | """ |
---|
6 | ftp_stat_cache.py - cache for (l)stat data |
---|
7 | """ |
---|
8 | |
---|
9 | import time |
---|
10 | |
---|
11 | import ftputil.error |
---|
12 | import ftputil.lrucache |
---|
13 | |
---|
14 | |
---|
15 | # This module shouldn't be used by clients of the ftputil library. |
---|
16 | __all__ = [] |
---|
17 | |
---|
18 | |
---|
19 | class StatCache: |
---|
20 | """ |
---|
21 | Implement an LRU (least-recently-used) cache. |
---|
22 | |
---|
23 | `StatCache` objects have an attribute `max_age`. After this |
---|
24 | duration after _setting_ it a cache entry will expire. For |
---|
25 | example, if you code |
---|
26 | |
---|
27 | my_cache = StatCache() |
---|
28 | my_cache.max_age = 10 |
---|
29 | my_cache["/home"] = ... |
---|
30 | |
---|
31 | the value my_cache["/home"] can be retrieved for 10 seconds. After |
---|
32 | that, the entry will be treated as if it had never been in the |
---|
33 | cache and should be fetched again from the remote host. |
---|
34 | |
---|
35 | Note that the `__len__` method does no age tests and thus may |
---|
36 | include some or many already expired entries. |
---|
37 | """ |
---|
38 | |
---|
39 | # Disable "Badly implemented container" warning because of |
---|
40 | # "missing" `__delitem__`. |
---|
41 | # pylint: disable=incomplete-protocol |
---|
42 | |
---|
43 | # Default number of cache entries |
---|
44 | _DEFAULT_CACHE_SIZE = 5000 |
---|
45 | |
---|
46 | def __init__(self): |
---|
47 | # Can be reset with method `resize` |
---|
48 | self._cache = ftputil.lrucache.LRUCache(self._DEFAULT_CACHE_SIZE) |
---|
49 | # Never expire |
---|
50 | self.max_age = None |
---|
51 | self.enable() |
---|
52 | |
---|
53 | def enable(self): |
---|
54 | """Enable storage of stat results.""" |
---|
55 | self._enabled = True |
---|
56 | |
---|
57 | def disable(self): |
---|
58 | """ |
---|
59 | Disable the cache. Further storage attempts with `__setitem__` |
---|
60 | won't have any visible effect. |
---|
61 | |
---|
62 | Disabling the cache only effects new storage attempts. Values |
---|
63 | stored before calling `disable` can still be retrieved unless |
---|
64 | disturbed by a `resize` command or normal cache expiration. |
---|
65 | """ |
---|
66 | # `_enabled` is set via calling `enable` in the constructor. |
---|
67 | # pylint: disable=attribute-defined-outside-init |
---|
68 | self._enabled = False |
---|
69 | |
---|
70 | def resize(self, new_size): |
---|
71 | """ |
---|
72 | Set number of cache entries to the integer `new_size`. |
---|
73 | If the new size is smaller than the current cache size, |
---|
74 | relatively long-unused elements will be removed. |
---|
75 | """ |
---|
76 | self._cache.size = new_size |
---|
77 | |
---|
78 | def _age(self, path): |
---|
79 | """ |
---|
80 | Return the age of a cache entry for `path` in seconds. If |
---|
81 | the path isn't in the cache, raise a `CacheMissError`. |
---|
82 | """ |
---|
83 | try: |
---|
84 | return time.time() - self._cache.mtime(path) |
---|
85 | except ftputil.lrucache.CacheKeyError: |
---|
86 | raise ftputil.error.CacheMissError( |
---|
87 | "no entry for path {} in cache".format(path)) |
---|
88 | |
---|
89 | def clear(self): |
---|
90 | """Clear (invalidate) all cache entries.""" |
---|
91 | self._cache.clear() |
---|
92 | |
---|
93 | def invalidate(self, path): |
---|
94 | """ |
---|
95 | Invalidate the cache entry for the absolute `path` if present. |
---|
96 | After that, the stat result data for `path` can no longer be |
---|
97 | retrieved, as if it had never been stored. |
---|
98 | |
---|
99 | If no stat result for `path` is in the cache, do _not_ |
---|
100 | raise an exception. |
---|
101 | """ |
---|
102 | #XXX To be 100 % sure, this should be `host.sep`, but I don't |
---|
103 | # want to introduce a reference to the `FTPHost` object for |
---|
104 | # only that purpose. |
---|
105 | assert path.startswith("/"), ("{} must be an absolute path". |
---|
106 | format(path)) |
---|
107 | try: |
---|
108 | del self._cache[path] |
---|
109 | except ftputil.lrucache.CacheKeyError: |
---|
110 | # Ignore errors |
---|
111 | pass |
---|
112 | |
---|
113 | def __getitem__(self, path): |
---|
114 | """ |
---|
115 | Return the stat entry for the `path`. If there's no stored |
---|
116 | stat entry or the cache is disabled, raise `CacheMissError`. |
---|
117 | """ |
---|
118 | if not self._enabled: |
---|
119 | raise ftputil.error.CacheMissError("cache is disabled") |
---|
120 | # Possibly raise a `CacheMissError` in `_age` |
---|
121 | if (self.max_age is not None) and (self._age(path) > self.max_age): |
---|
122 | self.invalidate(path) |
---|
123 | raise ftputil.error.CacheMissError( |
---|
124 | "entry for path {} has expired".format(path)) |
---|
125 | else: |
---|
126 | #XXX I don't know if this may raise a `CacheMissError` in |
---|
127 | # case of race conditions. I prefer robust code. |
---|
128 | try: |
---|
129 | return self._cache[path] |
---|
130 | except ftputil.lrucache.CacheKeyError: |
---|
131 | raise ftputil.error.CacheMissError( |
---|
132 | "entry for path {} not found".format(path)) |
---|
133 | |
---|
134 | def __setitem__(self, path, stat_result): |
---|
135 | """ |
---|
136 | Put the stat data for the absolute `path` into the cache, |
---|
137 | unless it's disabled. |
---|
138 | """ |
---|
139 | assert path.startswith("/") |
---|
140 | if not self._enabled: |
---|
141 | return |
---|
142 | self._cache[path] = stat_result |
---|
143 | |
---|
144 | def __contains__(self, path): |
---|
145 | """ |
---|
146 | Support for the `in` operator. Return a true value, if data |
---|
147 | for `path` is in the cache, else return a false value. |
---|
148 | """ |
---|
149 | try: |
---|
150 | # Implicitly do an age test which may raise `CacheMissError`. |
---|
151 | self[path] |
---|
152 | except ftputil.error.CacheMissError: |
---|
153 | return False |
---|
154 | else: |
---|
155 | return True |
---|
156 | |
---|
157 | # |
---|
158 | # The following methods are only intended for debugging! |
---|
159 | # |
---|
160 | def __len__(self): |
---|
161 | """ |
---|
162 | Return the number of entries in the cache. Note that this |
---|
163 | may include some (or many) expired entries. |
---|
164 | """ |
---|
165 | return len(self._cache) |
---|
166 | |
---|
167 | def __str__(self): |
---|
168 | """Return a string representation of the cache contents.""" |
---|
169 | lines = [] |
---|
170 | for key in sorted(self._cache): |
---|
171 | lines.append("{}: {}".format(key, self[key])) |
---|
172 | return "\n".join(lines) |
---|