diff --git a/radicale/storage/multifilesystem/cache.py b/radicale/storage/multifilesystem/cache.py index 1d90f975..0f37381c 100644 --- a/radicale/storage/multifilesystem/cache.py +++ b/radicale/storage/multifilesystem/cache.py @@ -73,6 +73,10 @@ class CollectionPartCache(CollectionBase): _hash.update(raw_text) return _hash.hexdigest() + @staticmethod + def _item_cache_mtime_and_size(size: bytes, raw_text: bytes) -> str: + return str(storage.CACHE_VERSION.decode()) + "size=" + str(size) + ";mtime=" + str(raw_text) + def _item_cache_content(self, item: radicale_item.Item) -> CacheContent: return CacheContent(item.uid, item.etag, item.serialize(), item.name, item.component_name, *item.time_range) @@ -80,8 +84,11 @@ class CollectionPartCache(CollectionBase): def _store_item_cache(self, href: str, item: radicale_item.Item, cache_hash: str = "") -> CacheContent: if not cache_hash: - cache_hash = self._item_cache_hash( - item.serialize().encode(self._encoding)) + if self._storage._use_mtime_and_size_for_item_cache is True: + raise RuntimeError("_store_item_cache called without cache_hash is not supported if [storage] use_mtime_and_size_for_item_cache is True") + else: + cache_hash = self._item_cache_hash( + item.serialize().encode(self._encoding)) cache_folder = self._storage._get_collection_cache_subfolder(self._filesystem_path, ".Radicale.cache", "item") content = self._item_cache_content(item) self._storage._makedirs_synced(cache_folder) @@ -96,12 +103,20 @@ class CollectionPartCache(CollectionBase): def _load_item_cache(self, href: str, cache_hash: str ) -> Optional[CacheContent]: cache_folder = self._storage._get_collection_cache_subfolder(self._filesystem_path, ".Radicale.cache", "item") + path = os.path.join(cache_folder, href) try: - with open(os.path.join(cache_folder, href), "rb") as f: + with open(path, "rb") as f: hash_, *remainder = pickle.load(f) if hash_ and hash_ == cache_hash: + if self._storage._debug_cache_actions is True: + logger.debug("Item cache match : %r with hash %r", path, cache_hash) return CacheContent(*remainder) + else: + if self._storage._debug_cache_actions is True: + logger.debug("Item cache no match : %r with hash %r", path, cache_hash) except FileNotFoundError: + if self._storage._debug_cache_actions is True: + logger.debug("Item cache not found : %r with hash %r", path, cache_hash) pass except (pickle.UnpicklingError, ValueError) as e: logger.warning("Failed to load item cache entry %r in %r: %s", diff --git a/radicale/storage/multifilesystem/get.py b/radicale/storage/multifilesystem/get.py index 543f004a..f74c8fb6 100644 --- a/radicale/storage/multifilesystem/get.py +++ b/radicale/storage/multifilesystem/get.py @@ -80,11 +80,18 @@ class CollectionPartGet(CollectionPartCache, CollectionPartLock, raise # The hash of the component in the file system. This is used to check, # if the entry in the cache is still valid. - cache_hash = self._item_cache_hash(raw_text) - if self._storage._debug_cache_actions is True: - logger.debug("Check cache for: %r with hash %r", path, cache_hash) + if self._storage._use_mtime_and_size_for_item_cache is True: + cache_hash = self._item_cache_mtime_and_size(os.stat(path).st_size, os.stat(path).st_mtime_ns) + if self._storage._debug_cache_actions is True: + logger.debug("Item cache check for: %r with mtime and size %r", path, cache_hash) + else: + cache_hash = self._item_cache_hash(raw_text) + if self._storage._debug_cache_actions is True: + logger.debug("Item cache check for: %r with hash %r", path, cache_hash) cache_content = self._load_item_cache(href, cache_hash) if cache_content is None: + if self._storage._debug_cache_actions is True: + logger.debug("Item cache miss for: %r", path) with self._acquire_cache_lock("item"): # Lock the item cache to prevent multiple processes from # generating the same data in parallel. @@ -101,6 +108,8 @@ class CollectionPartGet(CollectionPartCache, CollectionPartLock, vobject_item, = vobject_items temp_item = radicale_item.Item( collection=self, vobject_item=vobject_item) + if self._storage._debug_cache_actions is True: + logger.debug("Item cache store for: %r", path) cache_content = self._store_item_cache( href, temp_item, cache_hash) except Exception as e: @@ -115,6 +124,9 @@ class CollectionPartGet(CollectionPartCache, CollectionPartLock, if not self._item_cache_cleaned: self._item_cache_cleaned = True self._clean_item_cache() + else: + if self._storage._debug_cache_actions is True: + logger.debug("Item cache hit for: %r", path) last_modified = time.strftime( "%a, %d %b %Y %H:%M:%S GMT", time.gmtime(os.path.getmtime(path))) diff --git a/radicale/storage/multifilesystem/upload.py b/radicale/storage/multifilesystem/upload.py index e9783e85..3814f428 100644 --- a/radicale/storage/multifilesystem/upload.py +++ b/radicale/storage/multifilesystem/upload.py @@ -41,19 +41,26 @@ class CollectionPartUpload(CollectionPartGet, CollectionPartCache, raise pathutils.UnsafePathError(href) path = pathutils.path_to_filesystem(self._filesystem_path, href) try: - cache_hash = self._item_cache_hash(item.serialize().encode(self._encoding)) - logger.debug("Store cache for: %r with hash %r", path, cache_hash) - self._store_item_cache(href, item, cache_hash) + with self._atomic_write(path, newline="") as fo: # type: ignore + f = cast(TextIO, fo) + f.write(item.serialize()) except Exception as e: raise ValueError("Failed to store item %r in collection %r: %s" % (href, self.path, e)) from e - # TODO: better fix for "mypy" - with self._atomic_write(path, newline="") as fo: # type: ignore - f = cast(TextIO, fo) - f.write(item.serialize()) - # Clean the cache after the actual item is stored, or the cache entry - # will be removed again. - self._clean_item_cache() + # store cache file + if self._storage._use_mtime_and_size_for_item_cache is True: + cache_hash = self._item_cache_mtime_and_size(os.stat(path).st_size, os.stat(path).st_mtime_ns) + if self._storage._debug_cache_actions is True: + logger.debug("Item cache store for: %r with mtime and size %r", path, cache_hash) + else: + cache_hash = self._item_cache_hash(item.serialize().encode(self._encoding)) + if self._storage._debug_cache_actions is True: + logger.debug("Item cache store for: %r with hash %r", path, cache_hash) + try: + self._store_item_cache(href, item, cache_hash) + except Exception as e: + raise ValueError("Failed to store item cache of %r in collection %r: %s" % + (href, self.path, e)) from e # Track the change self._update_history_etag(href, item) self._clean_history() @@ -84,15 +91,11 @@ class CollectionPartUpload(CollectionPartGet, CollectionPartCache, for item in items: uid = item.uid logger.debug("Store item from list with uid: '%s'" % uid) - try: - cache_content = self._item_cache_content(item) - except Exception as e: - raise ValueError( - "Failed to store item %r in temporary collection %r: %s" % - (uid, self.path, e)) from e + cache_content = self._item_cache_content(item) for href in get_safe_free_hrefs(uid): + path = os.path.join(self._filesystem_path, href) try: - f = open(os.path.join(self._filesystem_path, href), + f = open(path, "w", newline="", encoding=self._encoding) except OSError as e: if (sys.platform != "win32" and e.errno == errno.EINVAL or @@ -104,14 +107,31 @@ class CollectionPartUpload(CollectionPartGet, CollectionPartCache, else: raise RuntimeError("No href found for item %r in temporary " "collection %r" % (uid, self.path)) - with f: - f.write(item.serialize()) - f.flush() - self._storage._fsync(f) - with open(os.path.join(cache_folder, href), "wb") as fb: + + try: + with f: + f.write(item.serialize()) + f.flush() + self._storage._fsync(f) + except Exception as e: + raise ValueError( + "Failed to store item %r in temporary collection %r: %s" % + (uid, self.path, e)) from e + + # store cache file + if self._storage._use_mtime_and_size_for_item_cache is True: + cache_hash = self._item_cache_mtime_and_size(os.stat(path).st_size, os.stat(path).st_mtime_ns) + if self._storage._debug_cache_actions is True: + logger.debug("Item cache store for: %r with mtime and size %r", path, cache_hash) + else: cache_hash = self._item_cache_hash(item.serialize().encode(self._encoding)) - logger.debug("Store cache for: %r with hash %r", fb.name, cache_hash) - pickle.dump(cache_content, fb) + if self._storage._debug_cache_actions is True: + logger.debug("Item cache store for: %r with hash %r", path, cache_hash) + path_cache = os.path.join(cache_folder, href) + if self._storage._debug_cache_actions is True: + logger.debug("Item cache store into: %r", path_cache) + with open(os.path.join(cache_folder, href), "wb") as fb: + pickle.dump((cache_hash, *cache_content), fb) fb.flush() self._storage._fsync(fb) self._storage._sync_directory(cache_folder)