oss/Radicale
oss/Radicale
1
0
Fork 0
mirror of https://github.com/Kozea/Radicale.git synced 2025-10-18 22:01:59 +00:00
Code Wiki Activity

Synced with origin

Browse source
This commit is contained in:
Tuna Celik 2023-02-10 22:03:33 +01:00
parent 9b3bb2de2b
commit cf81d1f9a7
94 changed files with 5096 additions and 3560 deletions
Download patch file Download diff file Expand all files Collapse all files

140
radicale/storage/__init__.py
View file

@ -1,4 +1,4 @@
# This file is part of Radicale Server - Calendar Server
# This file is part of Radicale - CalDAV and CardDAV server
# Copyright © 2014 Jean-Marc Martins
# Copyright © 2012-2017 Guillaume Ayoub
# Copyright © 2017-2018 Unrud <unrud@outlook.com>
@ -23,37 +23,43 @@ Take a look at the class ``BaseCollection`` if you want to implement your own.
"""
import contextlib
import json
import xml.etree.ElementTree as ET
from hashlib import sha256
from typing import (Iterable, Iterator, Mapping, Optional, Sequence, Set,
Tuple, Union, overload)
import pkg_resources
import vobject
from radicale import utils
from radicale import config
from radicale import item as radicale_item
from radicale import types, utils
from radicale.item import filter as radicale_filter
INTERNAL_TYPES = ("multifilesystem",)
INTERNAL_TYPES: Sequence[str] = ("multifilesystem", "multifilesystem_nolock",)
CACHE_DEPS = ("radicale", "vobject", "python-dateutil",)
CACHE_VERSION = (";".join(pkg_resources.get_distribution(pkg).version
for pkg in CACHE_DEPS) + ";").encode()
CACHE_DEPS: Sequence[str] = ("radicale", "vobject", "python-dateutil",)
CACHE_VERSION: bytes = "".join(
"%s=%s;" % (pkg, utils.package_version(pkg))
for pkg in CACHE_DEPS).encode()
def load(configuration):
def load(configuration: "config.Configuration") -> "BaseStorage":
"""Load the storage module chosen in configuration."""
return utils.load_plugin(
INTERNAL_TYPES, "storage", "Storage", configuration)
return utils.load_plugin(INTERNAL_TYPES, "storage", "Storage", BaseStorage,
configuration)
class ComponentExistsError(ValueError):
def __init__(self, path):
def __init__(self, path: str) -> None:
message = "Component already exists: %r" % path
super().__init__(message)
class ComponentNotFoundError(ValueError):
def __init__(self, path):
def __init__(self, path: str) -> None:
message = "Component doesn't exist: %r" % path
super().__init__(message)
@ -61,47 +67,58 @@ class ComponentNotFoundError(ValueError):
class BaseCollection:
@property
def path(self):
def path(self) -> str:
"""The sanitized path of the collection without leading or
trailing ``/``."""
raise NotImplementedError
@property
def owner(self):
def owner(self) -> str:
"""The owner of the collection."""
return self.path.split("/", maxsplit=1)[0]
@property
def is_principal(self):
def is_principal(self) -> bool:
"""Collection is a principal."""
return bool(self.path) and "/" not in self.path
@property
def etag(self):
def etag(self) -> str:
"""Encoded as quoted-string (see RFC 2616)."""
etag = sha256()
for item in self.get_all():
assert item.href
etag.update((item.href + "/" + item.etag).encode())
etag.update(json.dumps(self.get_meta(), sort_keys=True).encode())
return '"%s"' % etag.hexdigest()
def sync(self, old_token=None):
@property
def tag(self) -> str:
"""The tag of the collection."""
return self.get_meta("tag") or ""
def sync(self, old_token: str = "") -> Tuple[str, Iterable[str]]:
"""Get the current sync token and changed items for synchronization.
``old_token`` an old sync token which is used as the base of the
delta update. If sync token is missing, all items are returned.
delta update. If sync token is empty, all items are returned.
ValueError is raised for invalid or old tokens.
WARNING: This simple default implementation treats all sync-token as
invalid.
"""
def hrefs_iter() -> Iterator[str]:
for item in self.get_all():
assert item.href
yield item.href
token = "http://radicale.org/ns/sync/%s" % self.etag.strip("\"")
if old_token:
raise ValueError("Sync token are not supported")
return token, (item.href for item in self.get_all())
return token, hrefs_iter()
def get_multi(self, hrefs):
def get_multi(self, hrefs: Iterable[str]
) -> Iterable[Tuple[str, Optional["radicale_item.Item"]]]:
"""Fetch multiple items.
It's not required to return the requested items in the correct order.
@ -113,11 +130,12 @@ class BaseCollection:
"""
raise NotImplementedError
def get_all(self):
def get_all(self) -> Iterable["radicale_item.Item"]:
"""Fetch all items."""
raise NotImplementedError
def get_filtered(self, filters):
def get_filtered(self, filters: Iterable[ET.Element]
) -> Iterable[Tuple["radicale_item.Item", bool]]:
"""Fetch all items with optional filtering.
This can largely improve performance of reports depending on
@ -128,32 +146,31 @@ class BaseCollection:
matched.
"""
if not self.tag:
return
tag, start, end, simple = radicale_filter.simplify_prefilters(
filters, collection_tag=self.get_meta("tag"))
filters, self.tag)
for item in self.get_all():
if tag:
if tag != item.component_name:
continue
istart, iend = item.time_range
if istart >= end or iend <= start:
continue
item_simple = simple and (start <= istart or iend <= end)
else:
item_simple = simple
yield item, item_simple
if tag is not None and tag != item.component_name:
continue
istart, iend = item.time_range
if istart >= end or iend <= start:
continue
yield item, simple and (start <= istart or iend <= end)
def has_uid(self, uid):
def has_uid(self, uid: str) -> bool:
"""Check if a UID exists in the collection."""
for item in self.get_all():
if item.uid == uid:
return True
return False
def upload(self, href, item):
def upload(self, href: str, item: "radicale_item.Item") -> (
"radicale_item.Item"):
"""Upload a new or replace an existing item."""
raise NotImplementedError
def delete(self, href=None):
def delete(self, href: Optional[str] = None) -> None:
"""Delete an item.
When ``href`` is ``None``, delete the collection.
@ -161,7 +178,14 @@ class BaseCollection:
"""
raise NotImplementedError
def get_meta(self, key=None):
@overload
def get_meta(self, key: None = None) -> Mapping[str, str]: ...
@overload
def get_meta(self, key: str) -> Optional[str]: ...
def get_meta(self, key: Optional[str] = None
) -> Union[Mapping[str, str], Optional[str]]:
"""Get metadata value for collection.
Return the value of the property ``key``. If ``key`` is ``None`` return
@ -170,7 +194,7 @@ class BaseCollection:
"""
raise NotImplementedError
def set_meta(self, props):
def set_meta(self, props: Mapping[str, str]) -> None:
"""Set metadata values for collection.
``props`` a dict with values for properties.
@ -179,23 +203,23 @@ class BaseCollection:
raise NotImplementedError
@property
def last_modified(self):
def last_modified(self) -> str:
"""Get the HTTP-datetime of when the collection was modified."""
raise NotImplementedError
def serialize(self):
def serialize(self) -> str:
"""Get the unicode string representing the whole collection."""
if self.get_meta("tag") == "VCALENDAR":
if self.tag == "VCALENDAR":
in_vcalendar = False
vtimezones = ""
included_tzids = set()
included_tzids: Set[str] = set()
vtimezone = []
tzid = None
components = ""
# Concatenate all child elements of VCALENDAR from all items
# together, while preventing duplicated VTIMEZONE entries.
# VTIMEZONEs are only distinguished by their TZID, if different
# timezones share the same TZID this produces errornous ouput.
# timezones share the same TZID this produces erroneous output.
# VObject fails at this too.
for item in self.get_all():
depth = 0
@ -216,6 +240,7 @@ class BaseCollection:
elif depth == 2 and line.startswith("END:"):
if tzid is None or tzid not in included_tzids:
vtimezones += "".join(vtimezone)
if tzid is not None:
included_tzids.add(tzid)
vtimezone.clear()
tzid = None
@ -240,13 +265,14 @@ class BaseCollection:
return (template[:template_insert_pos] +
vtimezones + components +
template[template_insert_pos:])
if self.get_meta("tag") == "VADDRESSBOOK":
if self.tag == "VADDRESSBOOK":
return "".join((item.serialize() for item in self.get_all()))
return ""
class BaseStorage:
def __init__(self, configuration):
def __init__(self, configuration: "config.Configuration") -> None:
"""Initialize BaseStorage.
``configuration`` see ``radicale.config`` module.
@ -256,7 +282,8 @@ class BaseStorage:
"""
self.configuration = configuration
def discover(self, path, depth="0"):
def discover(self, path: str, depth: str = "0") -> Iterable[
"types.CollectionOrItem"]:
"""Discover a list of collections under the given ``path``.
``path`` is sanitized.
@ -272,7 +299,8 @@ class BaseStorage:
"""
raise NotImplementedError
def move(self, item, to_collection, to_href):
def move(self, item: "radicale_item.Item", to_collection: BaseCollection,
to_href: str) -> None:
"""Move an object.
``item`` is the item to move.
@ -285,7 +313,10 @@ class BaseStorage:
"""
raise NotImplementedError
def create_collection(self, href, items=None, props=None):
def create_collection(
self, href: str,
items: Optional[Iterable["radicale_item.Item"]] = None,
props: Optional[Mapping[str, str]] = None) -> BaseCollection:
"""Create a collection.
``href`` is the sanitized path.
@ -298,15 +329,14 @@ class BaseStorage:
``props`` are metadata values for the collection.
``props["tag"]`` is the type of collection (VCALENDAR or
VADDRESSBOOK). If the key ``tag`` is missing, it is guessed from the
collection.
``props["tag"]`` is the type of collection (VCALENDAR or VADDRESSBOOK).
If the key ``tag`` is missing, ``items`` is ignored.
"""
raise NotImplementedError
@contextlib.contextmanager
def acquire_lock(self, mode, user=None):
@types.contextmanager
def acquire_lock(self, mode: str, user: str = "") -> Iterator[None]:
"""Set a context manager to lock the whole storage.
``mode`` must either be "r" for shared access or "w" for exclusive
@ -317,6 +347,6 @@ class BaseStorage:
"""
raise NotImplementedError
def verify(self):
def verify(self) -> bool:
"""Check the storage for errors."""
raise NotImplementedError

148
radicale/storage/multifilesystem/__init__.py
View file

@ -1,4 +1,4 @@
# This file is part of Radicale Server - Calendar Server
# This file is part of Radicale - CalDAV and CardDAV server
# Copyright © 2014 Jean-Marc Martins
# Copyright © 2012-2017 Guillaume Ayoub
# Copyright © 2017-2019 Unrud <unrud@outlook.com>
@ -23,75 +23,57 @@ Uses one folder per collection and one file per collection entry.
"""
import contextlib
import os
import time
from itertools import chain
from tempfile import TemporaryDirectory
from typing import ClassVar, Iterator, Optional, Type
from radicale import pathutils, storage
from radicale.storage.multifilesystem.cache import CollectionCacheMixin
from radicale import config
from radicale.storage.multifilesystem.base import CollectionBase, StorageBase
from radicale.storage.multifilesystem.cache import CollectionPartCache
from radicale.storage.multifilesystem.create_collection import \
StorageCreateCollectionMixin
from radicale.storage.multifilesystem.delete import CollectionDeleteMixin
from radicale.storage.multifilesystem.discover import StorageDiscoverMixin
from radicale.storage.multifilesystem.get import CollectionGetMixin
from radicale.storage.multifilesystem.history import CollectionHistoryMixin
from radicale.storage.multifilesystem.lock import (CollectionLockMixin,
StorageLockMixin)
from radicale.storage.multifilesystem.meta import CollectionMetaMixin
from radicale.storage.multifilesystem.move import StorageMoveMixin
from radicale.storage.multifilesystem.sync import CollectionSyncMixin
from radicale.storage.multifilesystem.upload import CollectionUploadMixin
from radicale.storage.multifilesystem.verify import StorageVerifyMixin
StoragePartCreateCollection
from radicale.storage.multifilesystem.delete import CollectionPartDelete
from radicale.storage.multifilesystem.discover import StoragePartDiscover
from radicale.storage.multifilesystem.get import CollectionPartGet
from radicale.storage.multifilesystem.history import CollectionPartHistory
from radicale.storage.multifilesystem.lock import (CollectionPartLock,
StoragePartLock)
from radicale.storage.multifilesystem.meta import CollectionPartMeta
from radicale.storage.multifilesystem.move import StoragePartMove
from radicale.storage.multifilesystem.sync import CollectionPartSync
from radicale.storage.multifilesystem.upload import CollectionPartUpload
from radicale.storage.multifilesystem.verify import StoragePartVerify
class Collection(
CollectionCacheMixin, CollectionDeleteMixin, CollectionGetMixin,
CollectionHistoryMixin, CollectionLockMixin, CollectionMetaMixin,
CollectionSyncMixin, CollectionUploadMixin, storage.BaseCollection):
CollectionPartDelete, CollectionPartMeta, CollectionPartSync,
CollectionPartUpload, CollectionPartGet, CollectionPartCache,
CollectionPartLock, CollectionPartHistory, CollectionBase):
def __init__(self, storage_, path, filesystem_path=None):
self._storage = storage_
folder = self._storage._get_collection_root_folder()
# Path should already be sanitized
self._path = pathutils.strip_path(path)
self._encoding = self._storage.configuration.get("encoding", "stock")
if filesystem_path is None:
filesystem_path = pathutils.path_to_filesystem(folder, self.path)
self._filesystem_path = filesystem_path
_etag_cache: Optional[str]
def __init__(self, storage_: "Storage", path: str,
filesystem_path: Optional[str] = None) -> None:
super().__init__(storage_, path, filesystem_path)
self._etag_cache = None
super().__init__()
@property
def path(self):
def path(self) -> str:
return self._path
@contextlib.contextmanager
def _atomic_write(self, path, mode="w", newline=None):
parent_dir, name = os.path.split(path)
# Do not use mkstemp because it creates with permissions 0o600
with TemporaryDirectory(
prefix=".Radicale.tmp-", dir=parent_dir) as tmp_dir:
with open(os.path.join(tmp_dir, name), mode, newline=newline,
encoding=None if "b" in mode else self._encoding) as tmp:
yield tmp
tmp.flush()
self._storage._fsync(tmp)
os.replace(os.path.join(tmp_dir, name), path)
self._storage._sync_directory(parent_dir)
@property
def last_modified(self):
relevant_files = chain(
(self._filesystem_path,),
(self._props_path,) if os.path.exists(self._props_path) else (),
(os.path.join(self._filesystem_path, h) for h in self._list()))
last = max(map(os.path.getmtime, relevant_files))
def last_modified(self) -> str:
def relevant_files_iter() -> Iterator[str]:
yield self._filesystem_path
if os.path.exists(self._props_path):
yield self._props_path
for href in self._list():
yield os.path.join(self._filesystem_path, href)
last = max(map(os.path.getmtime, relevant_files_iter()))
return time.strftime("%a, %d %b %Y %H:%M:%S GMT", time.gmtime(last))
@property
def etag(self):
def etag(self) -> str:
# reuse cached value if the storage is read-only
if self._storage._lock.locked == "w" or self._etag_cache is None:
self._etag_cache = super().etag
@ -99,61 +81,11 @@ class Collection(
class Storage(
StorageCreateCollectionMixin, StorageDiscoverMixin, StorageLockMixin,
StorageMoveMixin, StorageVerifyMixin, storage.BaseStorage):
StoragePartCreateCollection, StoragePartLock, StoragePartMove,
StoragePartVerify, StoragePartDiscover, StorageBase):
_collection_class = Collection
_collection_class: ClassVar[Type[Collection]] = Collection
def __init__(self, configuration):
def __init__(self, configuration: config.Configuration) -> None:
super().__init__(configuration)
folder = configuration.get("storage", "filesystem_folder")
self._makedirs_synced(folder)
def _get_collection_root_folder(self):
filesystem_folder = self.configuration.get(
"storage", "filesystem_folder")
return os.path.join(filesystem_folder, "collection-root")
def _fsync(self, f):
if self.configuration.get("storage", "_filesystem_fsync"):
try:
pathutils.fsync(f.fileno())
except OSError as e:
raise RuntimeError("Fsync'ing file %r failed: %s" %
(f.name, e)) from e
def _sync_directory(self, path):
"""Sync directory to disk.
This only works on POSIX and does nothing on other systems.
"""
if not self.configuration.get("storage", "_filesystem_fsync"):
return
if os.name == "posix":
try:
fd = os.open(path, 0)
try:
pathutils.fsync(fd)
finally:
os.close(fd)
except OSError as e:
raise RuntimeError("Fsync'ing directory %r failed: %s" %
(path, e)) from e
def _makedirs_synced(self, filesystem_path):
"""Recursively create a directory and its parents in a sync'ed way.
This method acts silently when the folder already exists.
"""
if os.path.isdir(filesystem_path):
return
parent_filesystem_path = os.path.dirname(filesystem_path)
# Prevent infinite loop
if filesystem_path != parent_filesystem_path:
# Create parent dirs recursively
self._makedirs_synced(parent_filesystem_path)
# Possible race!
os.makedirs(filesystem_path, exist_ok=True)
self._sync_directory(parent_filesystem_path)
self._makedirs_synced(self._filesystem_folder)

123
radicale/storage/multifilesystem/base.py Normal file
View file

@ -0,0 +1,123 @@
# This file is part of Radicale - CalDAV and CardDAV server
# Copyright © 2014 Jean-Marc Martins
# Copyright © 2012-2017 Guillaume Ayoub
# Copyright © 2017-2019 Unrud <unrud@outlook.com>
#
# This library is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This library is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Radicale. If not, see <http://www.gnu.org/licenses/>.
import os
import sys
from tempfile import TemporaryDirectory
from typing import IO, AnyStr, ClassVar, Iterator, Optional, Type
from radicale import config, pathutils, storage, types
from radicale.storage import multifilesystem # noqa:F401
class CollectionBase(storage.BaseCollection):
_storage: "multifilesystem.Storage"
_path: str
_encoding: str
_filesystem_path: str
def __init__(self, storage_: "multifilesystem.Storage", path: str,
filesystem_path: Optional[str] = None) -> None:
super().__init__()
self._storage = storage_
folder = storage_._get_collection_root_folder()
# Path should already be sanitized
self._path = pathutils.strip_path(path)
self._encoding = storage_.configuration.get("encoding", "stock")
if filesystem_path is None:
filesystem_path = pathutils.path_to_filesystem(folder, self.path)
self._filesystem_path = filesystem_path
@types.contextmanager
def _atomic_write(self, path: str, mode: str = "w",
newline: Optional[str] = None) -> Iterator[IO[AnyStr]]:
# TODO: Overload with Literal when dropping support for Python < 3.8
parent_dir, name = os.path.split(path)
# Do not use mkstemp because it creates with permissions 0o600
with TemporaryDirectory(
prefix=".Radicale.tmp-", dir=parent_dir) as tmp_dir:
with open(os.path.join(tmp_dir, name), mode, newline=newline,
encoding=None if "b" in mode else self._encoding) as tmp:
yield tmp
tmp.flush()
self._storage._fsync(tmp)
os.replace(os.path.join(tmp_dir, name), path)
self._storage._sync_directory(parent_dir)
class StorageBase(storage.BaseStorage):
_collection_class: ClassVar[Type["multifilesystem.Collection"]]
_filesystem_folder: str
_filesystem_fsync: bool
def __init__(self, configuration: config.Configuration) -> None:
super().__init__(configuration)
self._filesystem_folder = configuration.get(
"storage", "filesystem_folder")
self._filesystem_fsync = configuration.get(
"storage", "_filesystem_fsync")
def _get_collection_root_folder(self) -> str:
return os.path.join(self._filesystem_folder, "collection-root")
def _fsync(self, f: IO[AnyStr]) -> None:
if self._filesystem_fsync:
try:
pathutils.fsync(f.fileno())
except OSError as e:
raise RuntimeError("Fsync'ing file %r failed: %s" %
(f.name, e)) from e
def _sync_directory(self, path: str) -> None:
"""Sync directory to disk.
This only works on POSIX and does nothing on other systems.
"""
if not self._filesystem_fsync:
return
if sys.platform != "win32":
try:
fd = os.open(path, 0)
try:
pathutils.fsync(fd)
finally:
os.close(fd)
except OSError as e:
raise RuntimeError("Fsync'ing directory %r failed: %s" %
(path, e)) from e
def _makedirs_synced(self, filesystem_path: str) -> None:
"""Recursively create a directory and its parents in a sync'ed way.
This method acts silently when the folder already exists.
"""
if os.path.isdir(filesystem_path):
return
parent_filesystem_path = os.path.dirname(filesystem_path)
# Prevent infinite loop
if filesystem_path != parent_filesystem_path:
# Create parent dirs recursively
self._makedirs_synced(parent_filesystem_path)
# Possible race!
os.makedirs(filesystem_path, exist_ok=True)
self._sync_directory(parent_filesystem_path)

66
radicale/storage/multifilesystem/cache.py
View file

@ -1,4 +1,4 @@
# This file is part of Radicale Server - Calendar Server
# This file is part of Radicale - CalDAV and CardDAV server
# Copyright © 2014 Jean-Marc Martins
# Copyright © 2012-2017 Guillaume Ayoub
# Copyright © 2017-2018 Unrud <unrud@outlook.com>
@ -16,20 +16,32 @@
# You should have received a copy of the GNU General Public License
# along with Radicale. If not, see <http://www.gnu.org/licenses/>.
import contextlib
import os
import pickle
import time
from hashlib import sha256
from typing import BinaryIO, Iterable, NamedTuple, Optional, cast
import radicale.item as radicale_item
from radicale import pathutils, storage
from radicale.log import logger
from radicale.storage.multifilesystem.base import CollectionBase
CacheContent = NamedTuple("CacheContent", [
("uid", str), ("etag", str), ("text", str), ("name", str), ("tag", str),
("start", int), ("end", int)])
class CollectionCacheMixin:
def _clean_cache(self, folder, names, max_age=None):
class CollectionPartCache(CollectionBase):
def _clean_cache(self, folder: str, names: Iterable[str],
max_age: int = 0) -> None:
"""Delete all ``names`` in ``folder`` that are older than ``max_age``.
"""
age_limit = time.time() - max_age if max_age is not None else None
age_limit: Optional[float] = None
if max_age is not None and max_age > 0:
age_limit = time.time() - max_age
modified = False
for name in names:
if not pathutils.is_safe_filesystem_path_component(name):
@ -54,51 +66,49 @@ class CollectionCacheMixin:
self._storage._sync_directory(folder)
@staticmethod
def _item_cache_hash(raw_text):
def _item_cache_hash(raw_text: bytes) -> str:
_hash = sha256()
_hash.update(storage.CACHE_VERSION)
_hash.update(raw_text)
return _hash.hexdigest()
def _item_cache_content(self, item, cache_hash=None):
text = item.serialize()
if cache_hash is None:
cache_hash = self._item_cache_hash(text.encode(self._encoding))
return (cache_hash, item.uid, item.etag, text, item.name,
item.component_name, *item.time_range)
def _item_cache_content(self, item: radicale_item.Item) -> CacheContent:
return CacheContent(item.uid, item.etag, item.serialize(), item.name,
item.component_name, *item.time_range)
def _store_item_cache(self, href, item, cache_hash=None):
def _store_item_cache(self, href: str, item: radicale_item.Item,
cache_hash: str = "") -> CacheContent:
if not cache_hash:
cache_hash = self._item_cache_hash(
item.serialize().encode(self._encoding))
cache_folder = os.path.join(self._filesystem_path, ".Radicale.cache",
"item")
content = self._item_cache_content(item, cache_hash)
content = self._item_cache_content(item)
self._storage._makedirs_synced(cache_folder)
try:
# Race: Other processes might have created and locked the
# file.
with self._atomic_write(os.path.join(cache_folder, href),
"wb") as f:
pickle.dump(content, f)
except PermissionError:
pass
# Race: Other processes might have created and locked the file.
with contextlib.suppress(PermissionError), self._atomic_write(
os.path.join(cache_folder, href), "wb") as fo:
fb = cast(BinaryIO, fo)
pickle.dump((cache_hash, *content), fb)
return content
def _load_item_cache(self, href, input_hash):
def _load_item_cache(self, href: str, cache_hash: str
) -> Optional[CacheContent]:
cache_folder = os.path.join(self._filesystem_path, ".Radicale.cache",
"item")
cache_hash = uid = etag = text = name = tag = start = end = None
try:
with open(os.path.join(cache_folder, href), "rb") as f:
cache_hash, *content = pickle.load(f)
if cache_hash == input_hash:
uid, etag, text, name, tag, start, end = content
hash_, *remainder = pickle.load(f)
if hash_ and hash_ == cache_hash:
return CacheContent(*remainder)
except FileNotFoundError:
pass
except (pickle.UnpicklingError, ValueError) as e:
logger.warning("Failed to load item cache entry %r in %r: %s",
href, self.path, e, exc_info=True)
return cache_hash, uid, etag, text, name, tag, start, end
return None
def _clean_item_cache(self):
def _clean_item_cache(self) -> None:
cache_folder = os.path.join(self._filesystem_path, ".Radicale.cache",
"item")
self._clean_cache(cache_folder, (

35
radicale/storage/multifilesystem/create_collection.py
View file

@ -1,4 +1,4 @@
# This file is part of Radicale Server - Calendar Server
# This file is part of Radicale - CalDAV and CardDAV server
# Copyright © 2014 Jean-Marc Martins
# Copyright © 2012-2017 Guillaume Ayoub
# Copyright © 2017-2018 Unrud <unrud@outlook.com>
@ -18,13 +18,19 @@
import os
from tempfile import TemporaryDirectory
from typing import Iterable, Optional, cast
import radicale.item as radicale_item
from radicale import pathutils
from radicale.storage import multifilesystem
from radicale.storage.multifilesystem.base import StorageBase
class StorageCreateCollectionMixin:
class StoragePartCreateCollection(StorageBase):
def create_collection(self, href, items=None, props=None):
def create_collection(self, href: str,
items: Optional[Iterable[radicale_item.Item]] = None,
props=None) -> "multifilesystem.Collection":
folder = self._get_collection_root_folder()
# Path should already be sanitized
@ -34,19 +40,21 @@ class StorageCreateCollectionMixin:
if not props:
self._makedirs_synced(filesystem_path)
return self._collection_class(
self, pathutils.unstrip_path(sane_path, True))
cast(multifilesystem.Storage, self),
pathutils.unstrip_path(sane_path, True))
parent_dir = os.path.dirname(filesystem_path)
self._makedirs_synced(parent_dir)
# Create a temporary directory with an unsafe name
with TemporaryDirectory(
prefix=".Radicale.tmp-", dir=parent_dir) as tmp_dir:
with TemporaryDirectory(prefix=".Radicale.tmp-", dir=parent_dir
) as tmp_dir:
# The temporary directory itself can't be renamed
tmp_filesystem_path = os.path.join(tmp_dir, "collection")
os.makedirs(tmp_filesystem_path)
col = self._collection_class(
self, pathutils.unstrip_path(sane_path, True),
cast(multifilesystem.Storage, self),
pathutils.unstrip_path(sane_path, True),
filesystem_path=tmp_filesystem_path)
col.set_meta(props)
if items is not None:
@ -55,13 +63,12 @@ class StorageCreateCollectionMixin:
elif props.get("tag") == "VADDRESSBOOK":
col._upload_all_nonatomic(items, suffix=".vcf")
# This operation is not atomic on the filesystem level but it's
# very unlikely that one rename operations succeeds while the
# other fails or that only one gets written to disk.
if os.path.exists(filesystem_path):
os.rename(filesystem_path, os.path.join(tmp_dir, "delete"))
os.rename(tmp_filesystem_path, filesystem_path)
if os.path.lexists(filesystem_path):
pathutils.rename_exchange(tmp_filesystem_path, filesystem_path)
else:
os.rename(tmp_filesystem_path, filesystem_path)
self._sync_directory(parent_dir)
return self._collection_class(
self, pathutils.unstrip_path(sane_path, True))
cast(multifilesystem.Storage, self),
pathutils.unstrip_path(sane_path, True))

14
radicale/storage/multifilesystem/delete.py
View file

@ -1,4 +1,4 @@
# This file is part of Radicale Server - Calendar Server
# This file is part of Radicale - CalDAV and CardDAV server
# Copyright © 2014 Jean-Marc Martins
# Copyright © 2012-2017 Guillaume Ayoub
# Copyright © 2017-2018 Unrud <unrud@outlook.com>
@ -18,20 +18,24 @@
import os
from tempfile import TemporaryDirectory
from typing import Optional
from radicale import pathutils, storage
from radicale.storage.multifilesystem.base import CollectionBase
from radicale.storage.multifilesystem.history import CollectionPartHistory
class CollectionDeleteMixin:
def delete(self, href=None):
class CollectionPartDelete(CollectionPartHistory, CollectionBase):
def delete(self, href: Optional[str] = None) -> None:
if href is None:
# Delete the collection
parent_dir = os.path.dirname(self._filesystem_path)
try:
os.rmdir(self._filesystem_path)
except OSError:
with TemporaryDirectory(
prefix=".Radicale.tmp-", dir=parent_dir) as tmp:
with TemporaryDirectory(prefix=".Radicale.tmp-", dir=parent_dir
) as tmp:
os.rename(self._filesystem_path, os.path.join(
tmp, os.path.basename(self._filesystem_path)))
self._storage._sync_directory(parent_dir)

42
radicale/storage/multifilesystem/discover.py
View file

@ -1,4 +1,4 @@
# This file is part of Radicale Server - Calendar Server
# This file is part of Radicale - CalDAV and CardDAV server
# Copyright © 2014 Jean-Marc Martins
# Copyright © 2012-2017 Guillaume Ayoub
# Copyright © 2017-2018 Unrud <unrud@outlook.com>
@ -16,18 +16,31 @@
# You should have received a copy of the GNU General Public License
# along with Radicale. If not, see <http://www.gnu.org/licenses/>.
import contextlib
import os
import posixpath
from typing import Callable, ContextManager, Iterator, Optional, cast
from radicale import pathutils
from radicale import pathutils, types
from radicale.log import logger
from radicale.storage import multifilesystem
from radicale.storage.multifilesystem.base import StorageBase
class StorageDiscoverMixin:
@types.contextmanager
def _null_child_context_manager(path: str,
href: Optional[str]) -> Iterator[None]:
yield
def discover(self, path, depth="0", child_context_manager=(
lambda path, href=None: contextlib.ExitStack())):
class StoragePartDiscover(StorageBase):
def discover(
self, path: str, depth: str = "0", child_context_manager: Optional[
Callable[[str, Optional[str]], ContextManager[None]]] = None
) -> Iterator[types.CollectionOrItem]:
# assert isinstance(self, multifilesystem.Storage)
if child_context_manager is None:
child_context_manager = _null_child_context_manager
# Path should already be sanitized
sane_path = pathutils.strip_path(path)
attributes = sane_path.split("/") if sane_path else []
@ -44,6 +57,7 @@ class StorageDiscoverMixin:
return
# Check if the path exists and if it leads to a collection or an item
href: Optional[str]
if not os.path.isdir(filesystem_path):
if attributes and os.path.isfile(filesystem_path):
href = attributes.pop()
@ -54,10 +68,13 @@ class StorageDiscoverMixin:
sane_path = "/".join(attributes)
collection = self._collection_class(
self, pathutils.unstrip_path(sane_path, True))
cast(multifilesystem.Storage, self),
pathutils.unstrip_path(sane_path, True))
if href:
yield collection._get(href)
item = collection._get(href)
if item is not None:
yield item
return
yield collection
@ -67,7 +84,9 @@ class StorageDiscoverMixin:
for href in collection._list():
with child_context_manager(sane_path, href):
yield collection._get(href)
item = collection._get(href)
if item is not None:
yield item
for entry in os.scandir(filesystem_path):
if not entry.is_dir():
@ -80,5 +99,6 @@ class StorageDiscoverMixin:
continue
sane_child_path = posixpath.join(sane_path, href)
child_path = pathutils.unstrip_path(sane_child_path, True)
with child_context_manager(sane_child_path):
yield self._collection_class(self, child_path)
with child_context_manager(sane_child_path, None):
yield self._collection_class(
cast(multifilesystem.Storage, self), child_path)

83
radicale/storage/multifilesystem/get.py
View file

@ -1,4 +1,4 @@
# This file is part of Radicale Server - Calendar Server
# This file is part of Radicale - CalDAV and CardDAV server
# Copyright © 2014 Jean-Marc Martins
# Copyright © 2012-2017 Guillaume Ayoub
# Copyright © 2017-2018 Unrud <unrud@outlook.com>
@ -17,21 +17,30 @@
# along with Radicale. If not, see <http://www.gnu.org/licenses/>.
import os
import sys
import time
from typing import Iterable, Iterator, Optional, Tuple
import vobject
from radicale import item as radicale_item
import radicale.item as radicale_item
from radicale import pathutils
from radicale.log import logger
from radicale.storage import multifilesystem
from radicale.storage.multifilesystem.base import CollectionBase
from radicale.storage.multifilesystem.cache import CollectionPartCache
from radicale.storage.multifilesystem.lock import CollectionPartLock
class CollectionGetMixin:
def __init__(self):
super().__init__()
class CollectionPartGet(CollectionPartCache, CollectionPartLock,
CollectionBase):
_item_cache_cleaned: bool
def __init__(self, storage_: "multifilesystem.Storage", path: str,
filesystem_path: Optional[str] = None) -> None:
super().__init__(storage_, path, filesystem_path)
self._item_cache_cleaned = False
def _list(self):
def _list(self) -> Iterator[str]:
for entry in os.scandir(self._filesystem_path):
if not entry.is_file():
continue
@ -42,13 +51,14 @@ class CollectionGetMixin:
continue
yield href
def _get(self, href, verify_href=True):
def _get(self, href: str, verify_href: bool = True
) -> Optional[radicale_item.Item]:
if verify_href:
try:
if not pathutils.is_safe_filesystem_path_component(href):
raise pathutils.UnsafePathError(href)
path = pathutils.path_to_filesystem(
self._filesystem_path, href)
path = pathutils.path_to_filesystem(self._filesystem_path,
href)
except ValueError as e:
logger.debug(
"Can't translate name %r safely to filesystem in %r: %s",
@ -63,36 +73,33 @@ class CollectionGetMixin:
return None
except PermissionError:
# Windows raises ``PermissionError`` when ``path`` is a directory
if (os.name == "nt" and
if (sys.platform == "win32" and
os.path.isdir(path) and os.access(path, os.R_OK)):
return None
raise
# The hash of the component in the file system. This is used to check,
# if the entry in the cache is still valid.
input_hash = self._item_cache_hash(raw_text)
cache_hash, uid, etag, text, name, tag, start, end = \
self._load_item_cache(href, input_hash)
if input_hash != cache_hash:
cache_hash = self._item_cache_hash(raw_text)
cache_content = self._load_item_cache(href, cache_hash)
if cache_content is None:
with self._acquire_cache_lock("item"):
# Lock the item cache to prevent multpile processes from
# generating the same data in parallel.
# This improves the performance for multiple requests.
if self._storage._lock.locked == "r":
# Check if another process created the file in the meantime
cache_hash, uid, etag, text, name, tag, start, end = \
self._load_item_cache(href, input_hash)
if input_hash != cache_hash:
cache_content = self._load_item_cache(href, cache_hash)
if cache_content is None:
try:
vobject_items = tuple(vobject.readComponents(
raw_text.decode(self._encoding)))
vobject_items = radicale_item.read_components(
raw_text.decode(self._encoding))
radicale_item.check_and_sanitize_items(
vobject_items, tag=self.get_meta("tag"))
vobject_items, tag=self.tag)
vobject_item, = vobject_items
temp_item = radicale_item.Item(
collection=self, vobject_item=vobject_item)
cache_hash, uid, etag, text, name, tag, start, end = \
self._store_item_cache(
href, temp_item, input_hash)
cache_content = self._store_item_cache(
href, temp_item, cache_hash)
except Exception as e:
raise RuntimeError("Failed to load item %r in %r: %s" %
(href, self.path, e)) from e
@ -107,11 +114,14 @@ class CollectionGetMixin:
# Don't keep reference to ``vobject_item``, because it requires a lot
# of memory.
return radicale_item.Item(
collection=self, href=href, last_modified=last_modified, etag=etag,
text=text, uid=uid, name=name, component_name=tag,
time_range=(start, end))
collection=self, href=href, last_modified=last_modified,
etag=cache_content.etag, text=cache_content.text,
uid=cache_content.uid, name=cache_content.name,
component_name=cache_content.tag,
time_range=(cache_content.start, cache_content.end))
def get_multi(self, hrefs):
def get_multi(self, hrefs: Iterable[str]
) -> Iterator[Tuple[str, Optional[radicale_item.Item]]]:
# It's faster to check for file name collissions here, because
# we only need to call os.listdir once.
files = None
@ -123,13 +133,16 @@ class CollectionGetMixin:
path = os.path.join(self._filesystem_path, href)
if (not pathutils.is_safe_filesystem_path_component(href) or
href not in files and os.path.lexists(path)):
logger.debug(
"Can't translate name safely to filesystem: %r", href)
logger.debug("Can't translate name safely to filesystem: %r",
href)
yield (href, None)
else:
yield (href, self._get(href, verify_href=False))
def get_all(self):
# We don't need to check for collissions, because the the file names
# are from os.listdir.
return (self._get(href, verify_href=False) for href in self._list())
def get_all(self) -> Iterator[radicale_item.Item]:
for href in self._list():
# We don't need to check for collissions, because the file names
# are from os.listdir.
item = self._get(href, verify_href=False)
if item is not None:
yield item

38
radicale/storage/multifilesystem/history.py
View file

@ -1,4 +1,4 @@
# This file is part of Radicale Server - Calendar Server
# This file is part of Radicale - CalDAV and CardDAV server
# Copyright © 2014 Jean-Marc Martins
# Copyright © 2012-2017 Guillaume Ayoub
# Copyright © 2017-2019 Unrud <unrud@outlook.com>
@ -17,15 +17,28 @@
# along with Radicale. If not, see <http://www.gnu.org/licenses/>.
import binascii
import contextlib
import os
import pickle
from typing import BinaryIO, Optional, cast
from radicale import item as radicale_item
import radicale.item as radicale_item
from radicale import pathutils
from radicale.log import logger
from radicale.storage import multifilesystem
from radicale.storage.multifilesystem.base import CollectionBase
class CollectionHistoryMixin:
class CollectionPartHistory(CollectionBase):
_max_sync_token_age: int
def __init__(self, storage_: "multifilesystem.Storage", path: str,
filesystem_path: Optional[str] = None) -> None:
super().__init__(storage_, path, filesystem_path)
self._max_sync_token_age = storage_.configuration.get(
"storage", "max_sync_token_age")
def _update_history_etag(self, href, item):
"""Updates and retrieves the history etag from the history cache.
@ -53,13 +66,11 @@ class CollectionHistoryMixin:
self._storage._makedirs_synced(history_folder)
history_etag = radicale_item.get_etag(
history_etag + "/" + etag).strip("\"")
try:
# Race: Other processes might have created and locked the file.
with self._atomic_write(os.path.join(history_folder, href),
"wb") as f:
pickle.dump([etag, history_etag], f)
except PermissionError:
pass
# Race: Other processes might have created and locked the file.
with contextlib.suppress(PermissionError), self._atomic_write(
os.path.join(history_folder, href), "wb") as fo:
fb = cast(BinaryIO, fo)
pickle.dump([etag, history_etag], fb)
return history_etag
def _get_deleted_history_hrefs(self):
@ -67,7 +78,7 @@ class CollectionHistoryMixin:
history cache."""
history_folder = os.path.join(self._filesystem_path,
".Radicale.cache", "history")
try:
with contextlib.suppress(FileNotFoundError):
for entry in os.scandir(history_folder):
href = entry.name
if not pathutils.is_safe_filesystem_path_component(href):
@ -75,13 +86,10 @@ class CollectionHistoryMixin:
if os.path.isfile(os.path.join(self._filesystem_path, href)):
continue
yield href
except FileNotFoundError:
pass
def _clean_history(self):
# Delete all expired history entries of deleted items.
history_folder = os.path.join(self._filesystem_path,
".Radicale.cache", "history")
self._clean_cache(history_folder, self._get_deleted_history_hrefs(),
max_age=self._storage.configuration.get(
"storage", "max_sync_token_age"))
max_age=self._max_sync_token_age)

74
radicale/storage/multifilesystem/lock.py
View file

@ -1,4 +1,4 @@
# This file is part of Radicale Server - Calendar Server
# This file is part of Radicale - CalDAV and CardDAV server
# Copyright © 2014 Jean-Marc Martins
# Copyright © 2012-2017 Guillaume Ayoub
# Copyright © 2017-2019 Unrud <unrud@outlook.com>
@ -20,52 +20,82 @@ import contextlib
import logging
import os
import shlex
import signal
import subprocess
import sys
from typing import Iterator
from radicale import pathutils
from radicale import config, pathutils, types
from radicale.log import logger
from radicale.storage.multifilesystem.base import CollectionBase, StorageBase
class CollectionLockMixin:
def _acquire_cache_lock(self, ns=""):
class CollectionPartLock(CollectionBase):
@types.contextmanager
def _acquire_cache_lock(self, ns: str = "") -> Iterator[None]:
if self._storage._lock.locked == "w":
return contextlib.ExitStack()
yield
return
cache_folder = os.path.join(self._filesystem_path, ".Radicale.cache")
self._storage._makedirs_synced(cache_folder)
lock_path = os.path.join(cache_folder,
".Radicale.lock" + (".%s" % ns if ns else ""))
lock = pathutils.RwLock(lock_path)
return lock.acquire("w")
with lock.acquire("w"):
yield
class StorageLockMixin:
class StoragePartLock(StorageBase):
def __init__(self, configuration):
_lock: pathutils.RwLock
_hook: str
def __init__(self, configuration: config.Configuration) -> None:
super().__init__(configuration)
folder = self.configuration.get("storage", "filesystem_folder")
lock_path = os.path.join(folder, ".Radicale.lock")
lock_path = os.path.join(self._filesystem_folder, ".Radicale.lock")
self._lock = pathutils.RwLock(lock_path)
self._hook = configuration.get("storage", "hook")
@contextlib.contextmanager
def acquire_lock(self, mode, user=None):
@types.contextmanager
def acquire_lock(self, mode: str, user: str = "") -> Iterator[None]:
with self._lock.acquire(mode):
yield
# execute hook
hook = self.configuration.get("storage", "hook")
if mode == "w" and hook:
folder = self.configuration.get("storage", "filesystem_folder")
logger.debug("Running hook")
if mode == "w" and self._hook:
debug = logger.isEnabledFor(logging.DEBUG)
# Use new process group for child to prevent terminals
# from sending SIGINT etc.
preexec_fn = None
creationflags = 0
if sys.platform == "win32":
creationflags |= subprocess.CREATE_NEW_PROCESS_GROUP
else:
# Process group is also used to identify child processes
preexec_fn = os.setpgrp
command = self._hook % {
"user": shlex.quote(user or "Anonymous")}
logger.debug("Running storage hook")
p = subprocess.Popen(
hook % {"user": shlex.quote(user or "Anonymous")},
stdin=subprocess.DEVNULL,
command, stdin=subprocess.DEVNULL,
stdout=subprocess.PIPE if debug else subprocess.DEVNULL,
stderr=subprocess.PIPE if debug else subprocess.DEVNULL,
shell=True, universal_newlines=True, cwd=folder)
stdout_data, stderr_data = p.communicate()
shell=True, universal_newlines=True, preexec_fn=preexec_fn,
cwd=self._filesystem_folder, creationflags=creationflags)
try:
stdout_data, stderr_data = p.communicate()
except BaseException: # e.g. KeyboardInterrupt or SystemExit
p.kill()
p.wait()
raise
finally:
if sys.platform != "win32":
# Kill remaining children identified by process group
with contextlib.suppress(OSError):
os.killpg(p.pid, signal.SIGKILL)
if stdout_data:
logger.debug("Captured stdout hook:\n%s", stdout_data)
logger.debug("Captured stdout from hook:\n%s", stdout_data)
if stderr_data:
logger.debug("Captured stderr hook:\n%s", stderr_data)
logger.debug("Captured stderr from hook:\n%s", stderr_data)
if p.returncode != 0:
raise subprocess.CalledProcessError(p.returncode, p.args)

41
radicale/storage/multifilesystem/meta.py
View file

@ -1,4 +1,4 @@
# This file is part of Radicale Server - Calendar Server
# This file is part of Radicale - CalDAV and CardDAV server
# Copyright © 2014 Jean-Marc Martins
# Copyright © 2012-2017 Guillaume Ayoub
# Copyright © 2017-2018 Unrud <unrud@outlook.com>
@ -18,32 +18,49 @@
import json
import os
from typing import Mapping, Optional, TextIO, Union, cast, overload
from radicale import item as radicale_item
import radicale.item as radicale_item
from radicale.storage import multifilesystem
from radicale.storage.multifilesystem.base import CollectionBase
class CollectionMetaMixin:
def __init__(self):
super().__init__()
class CollectionPartMeta(CollectionBase):
_meta_cache: Optional[Mapping[str, str]]
_props_path: str
def __init__(self, storage_: "multifilesystem.Storage", path: str,
filesystem_path: Optional[str] = None) -> None:
super().__init__(storage_, path, filesystem_path)
self._meta_cache = None
self._props_path = os.path.join(
self._filesystem_path, ".Radicale.props")
def get_meta(self, key=None):
@overload
def get_meta(self, key: None = None) -> Mapping[str, str]: ...
@overload
def get_meta(self, key: str) -> Optional[str]: ...
def get_meta(self, key: Optional[str] = None) -> Union[Mapping[str, str],
Optional[str]]:
# reuse cached value if the storage is read-only
if self._storage._lock.locked == "w" or self._meta_cache is None:
try:
try:
with open(self._props_path, encoding=self._encoding) as f:
self._meta_cache = json.load(f)
temp_meta = json.load(f)
except FileNotFoundError:
self._meta_cache = {}
radicale_item.check_and_sanitize_props(self._meta_cache)
temp_meta = {}
self._meta_cache = radicale_item.check_and_sanitize_props(
temp_meta)
except ValueError as e:
raise RuntimeError("Failed to load properties of collection "
"%r: %s" % (self.path, e)) from e
return self._meta_cache.get(key) if key else self._meta_cache
return self._meta_cache if key is None else self._meta_cache.get(key)
def set_meta(self, props):
with self._atomic_write(self._props_path, "w") as f:
def set_meta(self, props: Mapping[str, str]) -> None:
with self._atomic_write(self._props_path, "w") as fo:
f = cast(TextIO, fo)
json.dump(props, f, sort_keys=True)

24
radicale/storage/multifilesystem/move.py
View file

@ -1,4 +1,4 @@
# This file is part of Radicale Server - Calendar Server
# This file is part of Radicale - CalDAV and CardDAV server
# Copyright © 2014 Jean-Marc Martins
# Copyright © 2012-2017 Guillaume Ayoub
# Copyright © 2017-2018 Unrud <unrud@outlook.com>
@ -18,19 +18,25 @@
import os
from radicale import pathutils
from radicale import item as radicale_item
from radicale import pathutils, storage
from radicale.storage import multifilesystem
from radicale.storage.multifilesystem.base import StorageBase
class StorageMoveMixin:
class StoragePartMove(StorageBase):
def move(self, item, to_collection, to_href):
def move(self, item: radicale_item.Item,
to_collection: storage.BaseCollection, to_href: str) -> None:
if not pathutils.is_safe_filesystem_path_component(to_href):
raise pathutils.UnsafePathError(to_href)
os.replace(
pathutils.path_to_filesystem(
item.collection._filesystem_path, item.href),
pathutils.path_to_filesystem(
to_collection._filesystem_path, to_href))
assert isinstance(to_collection, multifilesystem.Collection)
assert isinstance(item.collection, multifilesystem.Collection)
assert item.href
os.replace(pathutils.path_to_filesystem(
item.collection._filesystem_path, item.href),
pathutils.path_to_filesystem(
to_collection._filesystem_path, to_href))
self._sync_directory(to_collection._filesystem_path)
if item.collection._filesystem_path != to_collection._filesystem_path:
self._sync_directory(item.collection._filesystem_path)

34
radicale/storage/multifilesystem/sync.py
View file

@ -1,4 +1,4 @@
# This file is part of Radicale Server - Calendar Server
# This file is part of Radicale - CalDAV and CardDAV server
# Copyright © 2014 Jean-Marc Martins
# Copyright © 2012-2017 Guillaume Ayoub
# Copyright © 2017-2019 Unrud <unrud@outlook.com>
@ -16,20 +16,27 @@
# You should have received a copy of the GNU General Public License
# along with Radicale. If not, see <http://www.gnu.org/licenses/>.
import contextlib
import itertools
import os
import pickle
from hashlib import sha256
from typing import BinaryIO, Iterable, Tuple, cast
from radicale.log import logger
from radicale.storage.multifilesystem.base import CollectionBase
from radicale.storage.multifilesystem.cache import CollectionPartCache
from radicale.storage.multifilesystem.history import CollectionPartHistory
class CollectionSyncMixin:
def sync(self, old_token=None):
class CollectionPartSync(CollectionPartCache, CollectionPartHistory,
CollectionBase):
def sync(self, old_token: str = "") -> Tuple[str, Iterable[str]]:
# The sync token has the form http://radicale.org/ns/sync/TOKEN_NAME
# where TOKEN_NAME is the sha256 hash of all history etags of present
# and past items of the collection.
def check_token_name(token_name):
def check_token_name(token_name: str) -> bool:
if len(token_name) != 64:
return False
for c in token_name:
@ -37,7 +44,7 @@ class CollectionSyncMixin:
return False
return True
old_token_name = None
old_token_name = ""
if old_token:
# Extract the token name from the sync token
if not old_token.startswith("http://radicale.org/ns/sync/"):
@ -78,10 +85,9 @@ class CollectionSyncMixin:
"Failed to load stored sync token %r in %r: %s",
old_token_name, self.path, e, exc_info=True)
# Delete the damaged file
try:
with contextlib.suppress(FileNotFoundError,
PermissionError):
os.remove(old_token_path)
except (FileNotFoundError, PermissionError):
pass
raise ValueError("Token not found: %r" % old_token)
# write the new token state or update the modification time of
# existing token state
@ -89,23 +95,21 @@ class CollectionSyncMixin:
self._storage._makedirs_synced(token_folder)
try:
# Race: Other processes might have created and locked the file.
with self._atomic_write(token_path, "wb") as f:
pickle.dump(state, f)
with self._atomic_write(token_path, "wb") as fo:
fb = cast(BinaryIO, fo)
pickle.dump(state, fb)
except PermissionError:
pass
else:
# clean up old sync tokens and item cache
self._clean_cache(token_folder, os.listdir(token_folder),
max_age=self._storage.configuration.get(
"storage", "max_sync_token_age"))
max_age=self._max_sync_token_age)
self._clean_history()
else:
# Try to update the modification time
try:
with contextlib.suppress(FileNotFoundError):
# Race: Another process might have deleted the file.
os.utime(token_path)
except FileNotFoundError:
pass
changes = []
# Find all new, changed and deleted (that are still in the item cache)
# items

93
radicale/storage/multifilesystem/upload.py
View file

@ -1,4 +1,4 @@
# This file is part of Radicale Server - Calendar Server
# This file is part of Radicale - CalDAV and CardDAV server
# Copyright © 2014 Jean-Marc Martins
# Copyright © 2012-2017 Guillaume Ayoub
# Copyright © 2017-2018 Unrud <unrud@outlook.com>
@ -16,15 +16,25 @@
# You should have received a copy of the GNU General Public License
# along with Radicale. If not, see <http://www.gnu.org/licenses/>.
import errno
import os
import pickle
import sys
from typing import Iterable, Iterator, TextIO, cast
from radicale import item as radicale_item
import radicale.item as radicale_item
from radicale import pathutils
from radicale.storage.multifilesystem.base import CollectionBase
from radicale.storage.multifilesystem.cache import CollectionPartCache
from radicale.storage.multifilesystem.get import CollectionPartGet
from radicale.storage.multifilesystem.history import CollectionPartHistory
class CollectionUploadMixin:
def upload(self, href, item):
class CollectionPartUpload(CollectionPartGet, CollectionPartCache,
CollectionPartHistory, CollectionBase):
def upload(self, href: str, item: radicale_item.Item
) -> radicale_item.Item:
if not pathutils.is_safe_filesystem_path_component(href):
raise pathutils.UnsafePathError(href)
try:
@ -33,27 +43,40 @@ class CollectionUploadMixin:
raise ValueError("Failed to store item %r in collection %r: %s" %
(href, self.path, e)) from e
path = pathutils.path_to_filesystem(self._filesystem_path, href)
with self._atomic_write(path, newline="") as fd:
fd.write(item.serialize())
with self._atomic_write(path, newline="") as fo:
f = cast(TextIO, fo)
f.write(item.serialize())
# Clean the cache after the actual item is stored, or the cache entry
# will be removed again.
self._clean_item_cache()
# Track the change
self._update_history_etag(href, item)
self._clean_history()
return self._get(href, verify_href=False)
uploaded_item = self._get(href, verify_href=False)
if uploaded_item is None:
raise RuntimeError("Storage modified externally")
return uploaded_item
def _upload_all_nonatomic(self, items, suffix=""):
"""Upload a new set of items.
def _upload_all_nonatomic(self, items: Iterable[radicale_item.Item],
suffix: str = "") -> None:
"""Upload a new set of items non-atomic"""
def is_safe_free_href(href: str) -> bool:
return (pathutils.is_safe_filesystem_path_component(href) and
not os.path.lexists(
os.path.join(self._filesystem_path, href)))
This takes a list of vobject items and
uploads them nonatomic and without existence checks.
def get_safe_free_hrefs(uid: str) -> Iterator[str]:
for href in [uid if uid.lower().endswith(suffix.lower())
else uid + suffix,
radicale_item.get_etag(uid).strip('"') + suffix]:
if is_safe_free_href(href):
yield href
yield radicale_item.find_available_uid(
lambda href: not is_safe_free_href(href), suffix)
"""
cache_folder = os.path.join(self._filesystem_path,
".Radicale.cache", "item")
self._storage._makedirs_synced(cache_folder)
hrefs = set()
for item in items:
uid = item.uid
try:
@ -62,43 +85,27 @@ class CollectionUploadMixin:
raise ValueError(
"Failed to store item %r in temporary collection %r: %s" %
(uid, self.path, e)) from e
href_candidate_funtions = []
if os.name in ("nt", "posix"):
href_candidate_funtions.append(
lambda: uid if uid.lower().endswith(suffix.lower())
else uid + suffix)
href_candidate_funtions.extend((
lambda: radicale_item.get_etag(uid).strip('"') + suffix,
lambda: radicale_item.find_available_uid(hrefs.__contains__,
suffix)))
href = f = None
while href_candidate_funtions:
href = href_candidate_funtions.pop(0)()
if href in hrefs:
continue
if not pathutils.is_safe_filesystem_path_component(href):
if not href_candidate_funtions:
raise pathutils.UnsafePathError(href)
continue
for href in get_safe_free_hrefs(uid):
try:
f = open(pathutils.path_to_filesystem(
self._filesystem_path, href),
"w", newline="", encoding=self._encoding)
break
f = open(os.path.join(self._filesystem_path, href),
"w", newline="", encoding=self._encoding)
except OSError as e:
if href_candidate_funtions and (
os.name == "posix" and e.errno == 22 or
os.name == "nt" and e.errno == 123):
if (sys.platform != "win32" and e.errno == errno.EINVAL or
sys.platform == "win32" and e.errno == 123):
# not a valid filename
continue
raise
break
else:
raise RuntimeError("No href found for item %r in temporary "
"collection %r" % (uid, self.path))
with f:
f.write(item.serialize())
f.flush()
self._storage._fsync(f)
hrefs.add(href)
with open(os.path.join(cache_folder, href), "wb") as f:
pickle.dump(cache_content, f)
f.flush()
self._storage._fsync(f)
with open(os.path.join(cache_folder, href), "wb") as fb:
pickle.dump(cache_content, fb)
fb.flush()
self._storage._fsync(fb)
self._storage._sync_directory(cache_folder)
self._storage._sync_directory(self._filesystem_path)

32
radicale/storage/multifilesystem/verify.py
View file

@ -1,4 +1,4 @@
# This file is part of Radicale Server - Calendar Server
# This file is part of Radicale - CalDAV and CardDAV server
# Copyright © 2014 Jean-Marc Martins
# Copyright © 2012-2017 Guillaume Ayoub
# Copyright © 2017-2018 Unrud <unrud@outlook.com>
@ -16,23 +16,27 @@
# You should have received a copy of the GNU General Public License
# along with Radicale. If not, see <http://www.gnu.org/licenses/>.
import contextlib
from typing import Iterator, Optional, Set
from radicale import pathutils, storage
from radicale import pathutils, storage, types
from radicale.log import logger
from radicale.storage.multifilesystem.base import StorageBase
from radicale.storage.multifilesystem.discover import StoragePartDiscover
class StorageVerifyMixin:
def verify(self):
class StoragePartVerify(StoragePartDiscover, StorageBase):
def verify(self) -> bool:
item_errors = collection_errors = 0
@contextlib.contextmanager
def exception_cm(sane_path, href=None):
@types.contextmanager
def exception_cm(sane_path: str, href: Optional[str]
) -> Iterator[None]:
nonlocal item_errors, collection_errors
try:
yield
except Exception as e:
if href:
if href is not None:
item_errors += 1
name = "item %r in %r" % (href, sane_path)
else:
@ -45,13 +49,14 @@ class StorageVerifyMixin:
sane_path = remaining_sane_paths.pop(0)
path = pathutils.unstrip_path(sane_path, True)
logger.debug("Verifying collection %r", sane_path)
with exception_cm(sane_path):
with exception_cm(sane_path, None):
saved_item_errors = item_errors
collection = None
uids = set()
collection: Optional[storage.BaseCollection] = None
uids: Set[str] = set()
has_child_collections = False
for item in self.discover(path, "1", exception_cm):
if not collection:
assert isinstance(item, storage.BaseCollection)
collection = item
collection.get_meta()
continue
@ -65,10 +70,11 @@ class StorageVerifyMixin:
uids.add(item.uid)
logger.debug("Verified item %r in %r",
item.href, sane_path)
assert collection
if item_errors == saved_item_errors:
collection.sync()
if has_child_collections and collection.get_meta("tag"):
if has_child_collections and collection.tag:
logger.error("Invalid collection %r: %r must not have "
"child collections", sane_path,
collection.get_meta("tag"))
collection.tag)
return item_errors == 0 and collection_errors == 0

114
radicale/storage/multifilesystem_nolock.py Normal file
View file

@ -0,0 +1,114 @@
# This file is part of Radicale - CalDAV and CardDAV server
# Copyright © 2021 Unrud <unrud@outlook.com>
#
# This library is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This library is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Radicale. If not, see <http://www.gnu.org/licenses/>.
"""
The multifilesystem backend without file-based locking.
"""
import threading
from collections import deque
from typing import ClassVar, Deque, Dict, Hashable, Iterator, Type
from radicale import config, pathutils, types
from radicale.storage import multifilesystem
class RwLock(pathutils.RwLock):
_cond: threading.Condition
def __init__(self) -> None:
super().__init__("")
self._cond = threading.Condition(self._lock)
@types.contextmanager
def acquire(self, mode: str, user: str = "") -> Iterator[None]:
if mode not in "rw":
raise ValueError("Invalid mode: %r" % mode)
with self._cond:
self._cond.wait_for(lambda: not self._writer and (
mode == "r" or self._readers == 0))
if mode == "r":
self._readers += 1
else:
self._writer = True
try:
yield
finally:
with self._cond:
if mode == "r":
self._readers -= 1
self._writer = False
if self._readers == 0:
self._cond.notify_all()
class LockDict:
_lock: threading.Lock
_dict: Dict[Hashable, Deque[threading.Lock]]
def __init__(self) -> None:
self._lock = threading.Lock()
self._dict = {}
@types.contextmanager
def acquire(self, key: Hashable) -> Iterator[None]:
with self._lock:
waiters = self._dict.get(key)
if waiters is None:
self._dict[key] = waiters = deque()
wait = bool(waiters)
waiter = threading.Lock()
waiter.acquire()
waiters.append(waiter)
if wait:
waiter.acquire()
try:
yield
finally:
with self._lock:
assert waiters[0] is waiter and self._dict[key] is waiters
del waiters[0]
if waiters:
waiters[0].release()
else:
del self._dict[key]
class Collection(multifilesystem.Collection):
_storage: "Storage"
@types.contextmanager
def _acquire_cache_lock(self, ns: str = "") -> Iterator[None]:
if self._storage._lock.locked == "w":
yield
return
with self._storage._cache_lock.acquire((self.path, ns)):
yield
class Storage(multifilesystem.Storage):
_collection_class: ClassVar[Type[Collection]] = Collection
_cache_lock: LockDict
def __init__(self, configuration: config.Configuration) -> None:
super().__init__(configuration)
self._lock = RwLock()
self._cache_lock = LockDict()