From 8869b34470960ffd0cf92d1f2cc270a78ce398f7 Mon Sep 17 00:00:00 2001 From: Unrud Date: Tue, 28 Aug 2018 16:19:36 +0200 Subject: [PATCH] refactor --- radicale/__init__.py | 952 +------------ radicale/__main__.py | 1 + radicale/app/__init__.py | 376 ++++++ radicale/app/delete.py | 70 + radicale/app/get.py | 105 ++ radicale/app/head.py | 33 + radicale/app/mkcalendar.py | 80 ++ radicale/app/mkcol.py | 81 ++ radicale/app/move.py | 93 ++ radicale/app/options.py | 39 + radicale/app/propfind.py | 395 ++++++ radicale/app/proppatch.py | 126 ++ radicale/app/put.py | 230 ++++ radicale/app/report.py | 296 ++++ radicale/auth/__init__.py | 107 ++ radicale/{auth.py => auth/htpasswd.py} | 111 +- radicale/auth/http_x_remote_user.py | 25 + radicale/auth/none.py | 25 + radicale/auth/remote_user.py | 25 + radicale/config.py | 1 + radicale/httputils.py | 61 + radicale/item/__init__.py | 374 +++++ radicale/item/filter.py | 529 ++++++++ radicale/log.py | 1 + radicale/pathutils.py | 217 +++ radicale/rights.py | 188 --- radicale/rights/__init__.py | 84 ++ radicale/rights/authenticated.py | 35 + radicale/rights/from_file.py | 68 + radicale/rights/owner_only.py | 35 + radicale/rights/owner_write.py | 39 + radicale/server.py | 1 + radicale/storage/__init__.py | 357 +++++ .../multifilesystem.py} | 902 +------------ radicale/tests/__init__.py | 1 + radicale/tests/custom/auth.py | 1 + radicale/tests/custom/rights.py | 2 +- radicale/tests/custom/storage.py | 5 +- radicale/tests/helpers.py | 1 + radicale/tests/test_auth.py | 1 + radicale/tests/test_base.py | 1 + radicale/tests/test_rights.py | 2 +- radicale/web/__init__.py | 54 + radicale/{web.py => web/internal.py} | 63 +- radicale/web/{ => internal_data}/css/icon.png | Bin radicale/web/{ => internal_data}/css/main.css | 0 radicale/web/{ => internal_data}/fn.js | 2 +- radicale/web/{ => internal_data}/index.html | 0 radicale/web/none.py | 26 + radicale/xmlutils.py | 1198 +---------------- setup.py | 7 +- 51 files changed, 4091 insertions(+), 3335 deletions(-) create mode 100644 radicale/app/__init__.py create mode 100644 radicale/app/delete.py create mode 100644 radicale/app/get.py create mode 100644 radicale/app/head.py create mode 100644 radicale/app/mkcalendar.py create mode 100644 radicale/app/mkcol.py create mode 100644 radicale/app/move.py create mode 100644 radicale/app/options.py create mode 100644 radicale/app/propfind.py create mode 100644 radicale/app/proppatch.py create mode 100644 radicale/app/put.py create mode 100644 radicale/app/report.py create mode 100644 radicale/auth/__init__.py rename radicale/{auth.py => auth/htpasswd.py} (63%) create mode 100644 radicale/auth/http_x_remote_user.py create mode 100644 radicale/auth/none.py create mode 100644 radicale/auth/remote_user.py create mode 100644 radicale/httputils.py create mode 100644 radicale/item/__init__.py create mode 100644 radicale/item/filter.py create mode 100644 radicale/pathutils.py delete mode 100644 radicale/rights.py create mode 100644 radicale/rights/__init__.py create mode 100644 radicale/rights/authenticated.py create mode 100644 radicale/rights/from_file.py create mode 100644 radicale/rights/owner_only.py create mode 100644 radicale/rights/owner_write.py create mode 100644 radicale/storage/__init__.py rename radicale/{storage.py => storage/multifilesystem.py} (52%) create mode 100644 radicale/web/__init__.py rename radicale/{web.py => web/internal.py} (61%) rename radicale/web/{ => internal_data}/css/icon.png (100%) rename radicale/web/{ => internal_data}/css/main.css (100%) rename radicale/web/{ => internal_data}/fn.js (99%) rename radicale/web/{ => internal_data}/index.html (100%) create mode 100644 radicale/web/none.py diff --git a/radicale/__init__.py b/radicale/__init__.py index 3d59405a..4cab3f7e 100644 --- a/radicale/__init__.py +++ b/radicale/__init__.py @@ -2,6 +2,7 @@ # Copyright © 2008 Nicolas Kandel # Copyright © 2008 Pascal Halter # Copyright © 2008-2017 Guillaume Ayoub +# Copyright © 2017-2018 Unrud # # This library is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -23,963 +24,16 @@ Can be used with an external WSGI server or the built-in server. """ -import base64 -import contextlib -import datetime -import io -import itertools -import logging import os import pkg_resources -import posixpath -import pprint -import random -import socket -import sys import threading -import time -import zlib -from http import client -from urllib.parse import urlparse, quote -from xml.etree import ElementTree as ET -import vobject -from radicale import auth, config, log, rights, storage, web, xmlutils -from radicale.log import logger +from radicale import config, log +from radicale.app import Application VERSION = pkg_resources.get_distribution("radicale").version -NOT_ALLOWED = ( - client.FORBIDDEN, (("Content-Type", "text/plain"),), - "Access to the requested resource forbidden.") -FORBIDDEN = ( - client.FORBIDDEN, (("Content-Type", "text/plain"),), - "Action on the requested resource refused.") -BAD_REQUEST = ( - client.BAD_REQUEST, (("Content-Type", "text/plain"),), "Bad Request") -NOT_FOUND = ( - client.NOT_FOUND, (("Content-Type", "text/plain"),), - "The requested resource could not be found.") -CONFLICT = ( - client.CONFLICT, (("Content-Type", "text/plain"),), - "Conflict in the request.") -WEBDAV_PRECONDITION_FAILED = ( - client.CONFLICT, (("Content-Type", "text/plain"),), - "WebDAV precondition failed.") -METHOD_NOT_ALLOWED = ( - client.METHOD_NOT_ALLOWED, (("Content-Type", "text/plain"),), - "The method is not allowed on the requested resource.") -PRECONDITION_FAILED = ( - client.PRECONDITION_FAILED, - (("Content-Type", "text/plain"),), "Precondition failed.") -REQUEST_TIMEOUT = ( - client.REQUEST_TIMEOUT, (("Content-Type", "text/plain"),), - "Connection timed out.") -REQUEST_ENTITY_TOO_LARGE = ( - client.REQUEST_ENTITY_TOO_LARGE, (("Content-Type", "text/plain"),), - "Request body too large.") -REMOTE_DESTINATION = ( - client.BAD_GATEWAY, (("Content-Type", "text/plain"),), - "Remote destination not supported.") -DIRECTORY_LISTING = ( - client.FORBIDDEN, (("Content-Type", "text/plain"),), - "Directory listings are not supported.") -INTERNAL_SERVER_ERROR = ( - client.INTERNAL_SERVER_ERROR, (("Content-Type", "text/plain"),), - "A server error occurred. Please contact the administrator.") - -DAV_HEADERS = "1, 2, 3, calendar-access, addressbook, extended-mkcol" - - -class Application: - """WSGI application managing collections.""" - - def __init__(self, configuration): - """Initialize application.""" - super().__init__() - self.configuration = configuration - self.Auth = auth.load(configuration) - self.Collection = storage.load(configuration) - self.Rights = rights.load(configuration) - self.Web = web.load(configuration) - self.encoding = configuration.get("encoding", "request") - - def headers_log(self, environ): - """Sanitize headers for logging.""" - request_environ = dict(environ) - - # Mask passwords - mask_passwords = self.configuration.getboolean( - "logging", "mask_passwords") - authorization = request_environ.get("HTTP_AUTHORIZATION", "") - if mask_passwords and authorization.startswith("Basic"): - request_environ["HTTP_AUTHORIZATION"] = "Basic **masked**" - if request_environ.get("HTTP_COOKIE"): - request_environ["HTTP_COOKIE"] = "**masked**" - - return request_environ - - def decode(self, text, environ): - """Try to magically decode ``text`` according to given ``environ``.""" - # List of charsets to try - charsets = [] - - # First append content charset given in the request - content_type = environ.get("CONTENT_TYPE") - if content_type and "charset=" in content_type: - charsets.append( - content_type.split("charset=")[1].split(";")[0].strip()) - # Then append default Radicale charset - charsets.append(self.encoding) - # Then append various fallbacks - charsets.append("utf-8") - charsets.append("iso8859-1") - - # Try to decode - for charset in charsets: - try: - return text.decode(charset) - except UnicodeDecodeError: - pass - raise UnicodeDecodeError - - def collect_allowed_items(self, items, user): - """Get items from request that user is allowed to access.""" - for item in items: - if isinstance(item, storage.BaseCollection): - path = storage.sanitize_path("/%s/" % item.path) - if item.get_meta("tag"): - permissions = self.Rights.authorized(user, path, "rw") - target = "collection with tag %r" % item.path - else: - permissions = self.Rights.authorized(user, path, "RW") - target = "collection %r" % item.path - else: - path = storage.sanitize_path("/%s/" % item.collection.path) - permissions = self.Rights.authorized(user, path, "rw") - target = "item %r from %r" % (item.href, item.collection.path) - if rights.intersect_permissions(permissions, "Ww"): - permission = "w" - status = "write" - elif rights.intersect_permissions(permissions, "Rr"): - permission = "r" - status = "read" - else: - permission = "" - status = "NO" - logger.debug( - "%s has %s access to %s", - repr(user) if user else "anonymous user", status, target) - if permission: - yield item, permission - - def __call__(self, environ, start_response): - with log.register_stream(environ["wsgi.errors"]): - try: - status, headers, answers = self._handle_request(environ) - except Exception as e: - try: - method = str(environ["REQUEST_METHOD"]) - except Exception: - method = "unknown" - try: - path = str(environ.get("PATH_INFO", "")) - except Exception: - path = "" - logger.error("An exception occurred during %s request on %r: " - "%s", method, path, e, exc_info=True) - status, headers, answer = INTERNAL_SERVER_ERROR - answer = answer.encode("ascii") - status = "%d %s" % ( - status, client.responses.get(status, "Unknown")) - headers = [ - ("Content-Length", str(len(answer)))] + list(headers) - answers = [answer] - start_response(status, headers) - return answers - - def _handle_request(self, environ): - """Manage a request.""" - def response(status, headers=(), answer=None): - headers = dict(headers) - # Set content length - if answer: - if hasattr(answer, "encode"): - logger.debug("Response content:\n%s", answer) - headers["Content-Type"] += "; charset=%s" % self.encoding - answer = answer.encode(self.encoding) - accept_encoding = [ - encoding.strip() for encoding in - environ.get("HTTP_ACCEPT_ENCODING", "").split(",") - if encoding.strip()] - - if "gzip" in accept_encoding: - zcomp = zlib.compressobj(wbits=16 + zlib.MAX_WBITS) - answer = zcomp.compress(answer) + zcomp.flush() - headers["Content-Encoding"] = "gzip" - - headers["Content-Length"] = str(len(answer)) - - # Add extra headers set in configuration - if self.configuration.has_section("headers"): - for key in self.configuration.options("headers"): - headers[key] = self.configuration.get("headers", key) - - # Start response - time_end = datetime.datetime.now() - status = "%d %s" % ( - status, client.responses.get(status, "Unknown")) - logger.info( - "%s response status for %r%s in %.3f seconds: %s", - environ["REQUEST_METHOD"], environ.get("PATH_INFO", ""), - depthinfo, (time_end - time_begin).total_seconds(), status) - # Return response content - return status, list(headers.items()), [answer] if answer else [] - - remote_host = "unknown" - if environ.get("REMOTE_HOST"): - remote_host = repr(environ["REMOTE_HOST"]) - elif environ.get("REMOTE_ADDR"): - remote_host = environ["REMOTE_ADDR"] - if environ.get("HTTP_X_FORWARDED_FOR"): - remote_host = "%r (forwarded by %s)" % ( - environ["HTTP_X_FORWARDED_FOR"], remote_host) - remote_useragent = "" - if environ.get("HTTP_USER_AGENT"): - remote_useragent = " using %r" % environ["HTTP_USER_AGENT"] - depthinfo = "" - if environ.get("HTTP_DEPTH"): - depthinfo = " with depth %r" % environ["HTTP_DEPTH"] - time_begin = datetime.datetime.now() - logger.info( - "%s request for %r%s received from %s%s", - environ["REQUEST_METHOD"], environ.get("PATH_INFO", ""), depthinfo, - remote_host, remote_useragent) - headers = pprint.pformat(self.headers_log(environ)) - logger.debug("Request headers:\n%s", headers) - - # Let reverse proxies overwrite SCRIPT_NAME - if "HTTP_X_SCRIPT_NAME" in environ: - # script_name must be removed from PATH_INFO by the client. - unsafe_base_prefix = environ["HTTP_X_SCRIPT_NAME"] - logger.debug("Script name overwritten by client: %r", - unsafe_base_prefix) - else: - # SCRIPT_NAME is already removed from PATH_INFO, according to the - # WSGI specification. - unsafe_base_prefix = environ.get("SCRIPT_NAME", "") - # Sanitize base prefix - base_prefix = storage.sanitize_path(unsafe_base_prefix).rstrip("/") - logger.debug("Sanitized script name: %r", base_prefix) - # Sanitize request URI (a WSGI server indicates with an empty path, - # that the URL targets the application root without a trailing slash) - path = storage.sanitize_path(environ.get("PATH_INFO", "")) - logger.debug("Sanitized path: %r", path) - - # Get function corresponding to method - function = getattr(self, "do_%s" % environ["REQUEST_METHOD"].upper()) - - # If "/.well-known" is not available, clients query "/" - if path == "/.well-known" or path.startswith("/.well-known/"): - return response(*NOT_FOUND) - - # Ask authentication backend to check rights - login = password = "" - external_login = self.Auth.get_external_login(environ) - authorization = environ.get("HTTP_AUTHORIZATION", "") - if external_login: - login, password = external_login - login, password = login or "", password or "" - elif authorization.startswith("Basic"): - authorization = authorization[len("Basic"):].strip() - login, password = self.decode(base64.b64decode( - authorization.encode("ascii")), environ).split(":", 1) - - user = self.Auth.login(login, password) or "" if login else "" - if user and login == user: - logger.info("Successful login: %r", user) - elif user: - logger.info("Successful login: %r -> %r", login, user) - elif login: - logger.info("Failed login attempt: %r", login) - # Random delay to avoid timing oracles and bruteforce attacks - delay = self.configuration.getfloat("auth", "delay") - if delay > 0: - random_delay = delay * (0.5 + random.random()) - logger.debug("Sleeping %.3f seconds", random_delay) - time.sleep(random_delay) - - if user and not storage.is_safe_path_component(user): - # Prevent usernames like "user/calendar.ics" - logger.info("Refused unsafe username: %r", user) - user = "" - - # Create principal collection - if user: - principal_path = "/%s/" % user - if self.Rights.authorized(user, principal_path, "W"): - with self.Collection.acquire_lock("r", user): - principal = next( - self.Collection.discover(principal_path, depth="1"), - None) - if not principal: - with self.Collection.acquire_lock("w", user): - try: - self.Collection.create_collection(principal_path) - except ValueError as e: - logger.warning("Failed to create principal " - "collection %r: %s", user, e) - user = "" - else: - logger.warning("Access to principal path %r denied by " - "rights backend", principal_path) - - if self.configuration.getboolean("internal", "internal_server"): - # Verify content length - content_length = int(environ.get("CONTENT_LENGTH") or 0) - if content_length: - max_content_length = self.configuration.getint( - "server", "max_content_length") - if max_content_length and content_length > max_content_length: - logger.info("Request body too large: %d", content_length) - return response(*REQUEST_ENTITY_TOO_LARGE) - - if not login or user: - status, headers, answer = function( - environ, base_prefix, path, user) - if (status, headers, answer) == NOT_ALLOWED: - logger.info("Access to %r denied for %s", path, - repr(user) if user else "anonymous user") - else: - status, headers, answer = NOT_ALLOWED - - if ((status, headers, answer) == NOT_ALLOWED and not user and - not external_login): - # Unknown or unauthorized user - logger.debug("Asking client for authentication") - status = client.UNAUTHORIZED - realm = self.configuration.get("auth", "realm") - headers = dict(headers) - headers.update({ - "WWW-Authenticate": - "Basic realm=\"%s\"" % realm}) - - return response(status, headers, answer) - - def _access(self, user, path, permission, item=None): - if permission not in "rw": - raise ValueError("Invalid permission argument: %r" % permission) - if not item: - permissions = permission + permission.upper() - parent_permissions = permission - elif isinstance(item, storage.BaseCollection): - if item.get_meta("tag"): - permissions = permission - else: - permissions = permission.upper() - parent_permissions = "" - else: - permissions = "" - parent_permissions = permission - if permissions and self.Rights.authorized(user, path, permissions): - return True - if parent_permissions: - parent_path = storage.sanitize_path( - "/%s/" % posixpath.dirname(path.strip("/"))) - if self.Rights.authorized(user, parent_path, parent_permissions): - return True - return False - - def _read_raw_content(self, environ): - content_length = int(environ.get("CONTENT_LENGTH") or 0) - if not content_length: - return b"" - content = environ["wsgi.input"].read(content_length) - if len(content) < content_length: - raise RuntimeError("Request body too short: %d" % len(content)) - return content - - def _read_content(self, environ): - content = self.decode(self._read_raw_content(environ), environ) - logger.debug("Request content:\n%s", content) - return content - - def _read_xml_content(self, environ): - content = self.decode(self._read_raw_content(environ), environ) - if not content: - return None - try: - xml_content = ET.fromstring(content) - except ET.ParseError as e: - logger.debug("Request content (Invalid XML):\n%s", content) - raise RuntimeError("Failed to parse XML: %s" % e) from e - if logger.isEnabledFor(logging.DEBUG): - logger.debug("Request content:\n%s", - xmlutils.pretty_xml(xml_content)) - return xml_content - - def _write_xml_content(self, xml_content): - if logger.isEnabledFor(logging.DEBUG): - logger.debug("Response content:\n%s", - xmlutils.pretty_xml(xml_content)) - f = io.BytesIO() - ET.ElementTree(xml_content).write(f, encoding=self.encoding, - xml_declaration=True) - return f.getvalue() - - def _webdav_error_response(self, namespace, name, - status=WEBDAV_PRECONDITION_FAILED[0]): - """Generate XML error response.""" - headers = {"Content-Type": "text/xml; charset=%s" % self.encoding} - content = self._write_xml_content( - xmlutils.webdav_error(namespace, name)) - return status, headers, content - - def _propose_filename(self, collection): - """Propose a filename for a collection.""" - tag = collection.get_meta("tag") - if tag == "VADDRESSBOOK": - fallback_title = "Address book" - suffix = ".vcf" - elif tag == "VCALENDAR": - fallback_title = "Calendar" - suffix = ".ics" - else: - fallback_title = posixpath.basename(collection.path) - suffix = "" - title = collection.get_meta("D:displayname") or fallback_title - if title and not title.lower().endswith(suffix.lower()): - title += suffix - return title - - def _content_disposition_attachement(self, filename): - value = "attachement" - try: - encoded_filename = quote(filename, encoding=self.encoding) - except UnicodeEncodeError as e: - logger.warning("Failed to encode filename: %r", filename, - exc_info=True) - encoded_filename = "" - if encoded_filename: - value += "; filename*=%s''%s" % (self.encoding, encoded_filename) - return value - - def do_DELETE(self, environ, base_prefix, path, user): - """Manage DELETE request.""" - if not self._access(user, path, "w"): - return NOT_ALLOWED - with self.Collection.acquire_lock("w", user): - item = next(self.Collection.discover(path), None) - if not item: - return NOT_FOUND - if not self._access(user, path, "w", item): - return NOT_ALLOWED - if_match = environ.get("HTTP_IF_MATCH", "*") - if if_match not in ("*", item.etag): - # ETag precondition not verified, do not delete item - return PRECONDITION_FAILED - if isinstance(item, storage.BaseCollection): - xml_answer = xmlutils.delete(base_prefix, path, item) - else: - xml_answer = xmlutils.delete( - base_prefix, path, item.collection, item.href) - headers = {"Content-Type": "text/xml; charset=%s" % self.encoding} - return client.OK, headers, self._write_xml_content(xml_answer) - - def do_GET(self, environ, base_prefix, path, user): - """Manage GET request.""" - # Redirect to .web if the root URL is requested - if not path.strip("/"): - web_path = ".web" - if not environ.get("PATH_INFO"): - web_path = posixpath.join(posixpath.basename(base_prefix), - web_path) - return (client.FOUND, - {"Location": web_path, "Content-Type": "text/plain"}, - "Redirected to %s" % web_path) - # Dispatch .web URL to web module - if path == "/.web" or path.startswith("/.web/"): - return self.Web.get(environ, base_prefix, path, user) - if not self._access(user, path, "r"): - return NOT_ALLOWED - with self.Collection.acquire_lock("r", user): - item = next(self.Collection.discover(path), None) - if not item: - return NOT_FOUND - if not self._access(user, path, "r", item): - return NOT_ALLOWED - if isinstance(item, storage.BaseCollection): - tag = item.get_meta("tag") - if not tag: - return DIRECTORY_LISTING - content_type = xmlutils.MIMETYPES[tag] - content_disposition = self._content_disposition_attachement( - self._propose_filename(item)) - else: - content_type = xmlutils.OBJECT_MIMETYPES[item.name] - content_disposition = "" - headers = { - "Content-Type": content_type, - "Last-Modified": item.last_modified, - "ETag": item.etag} - if content_disposition: - headers["Content-Disposition"] = content_disposition - answer = item.serialize() - return client.OK, headers, answer - - def do_HEAD(self, environ, base_prefix, path, user): - """Manage HEAD request.""" - status, headers, answer = self.do_GET( - environ, base_prefix, path, user) - return status, headers, None - - def do_MKCALENDAR(self, environ, base_prefix, path, user): - """Manage MKCALENDAR request.""" - if not self.Rights.authorized(user, path, "w"): - return NOT_ALLOWED - try: - xml_content = self._read_xml_content(environ) - except RuntimeError as e: - logger.warning( - "Bad MKCALENDAR request on %r: %s", path, e, exc_info=True) - return BAD_REQUEST - except socket.timeout as e: - logger.debug("client timed out", exc_info=True) - return REQUEST_TIMEOUT - # Prepare before locking - props = xmlutils.props_from_request(xml_content) - props["tag"] = "VCALENDAR" - # TODO: use this? - # timezone = props.get("C:calendar-timezone") - try: - storage.check_and_sanitize_props(props) - except ValueError as e: - logger.warning( - "Bad MKCALENDAR request on %r: %s", path, e, exc_info=True) - with self.Collection.acquire_lock("w", user): - item = next(self.Collection.discover(path), None) - if item: - return self._webdav_error_response( - "D", "resource-must-be-null") - parent_path = storage.sanitize_path( - "/%s/" % posixpath.dirname(path.strip("/"))) - parent_item = next(self.Collection.discover(parent_path), None) - if not parent_item: - return CONFLICT - if (not isinstance(parent_item, storage.BaseCollection) or - parent_item.get_meta("tag")): - return FORBIDDEN - try: - self.Collection.create_collection(path, props=props) - except ValueError as e: - logger.warning( - "Bad MKCALENDAR request on %r: %s", path, e, exc_info=True) - return BAD_REQUEST - return client.CREATED, {}, None - - def do_MKCOL(self, environ, base_prefix, path, user): - """Manage MKCOL request.""" - permissions = self.Rights.authorized(user, path, "Ww") - if not permissions: - return NOT_ALLOWED - try: - xml_content = self._read_xml_content(environ) - except RuntimeError as e: - logger.warning( - "Bad MKCOL request on %r: %s", path, e, exc_info=True) - return BAD_REQUEST - except socket.timeout as e: - logger.debug("client timed out", exc_info=True) - return REQUEST_TIMEOUT - # Prepare before locking - props = xmlutils.props_from_request(xml_content) - try: - storage.check_and_sanitize_props(props) - except ValueError as e: - logger.warning( - "Bad MKCOL request on %r: %s", path, e, exc_info=True) - return BAD_REQUEST - if (props.get("tag") and "w" not in permissions or - not props.get("tag") and "W" not in permissions): - return NOT_ALLOWED - with self.Collection.acquire_lock("w", user): - item = next(self.Collection.discover(path), None) - if item: - return METHOD_NOT_ALLOWED - parent_path = storage.sanitize_path( - "/%s/" % posixpath.dirname(path.strip("/"))) - parent_item = next(self.Collection.discover(parent_path), None) - if not parent_item: - return CONFLICT - if (not isinstance(parent_item, storage.BaseCollection) or - parent_item.get_meta("tag")): - return FORBIDDEN - try: - self.Collection.create_collection(path, props=props) - except ValueError as e: - logger.warning( - "Bad MKCOL request on %r: %s", path, e, exc_info=True) - return BAD_REQUEST - return client.CREATED, {}, None - - def do_MOVE(self, environ, base_prefix, path, user): - """Manage MOVE request.""" - raw_dest = environ.get("HTTP_DESTINATION", "") - to_url = urlparse(raw_dest) - if to_url.netloc != environ["HTTP_HOST"]: - logger.info("Unsupported destination address: %r", raw_dest) - # Remote destination server, not supported - return REMOTE_DESTINATION - if not self._access(user, path, "w"): - return NOT_ALLOWED - to_path = storage.sanitize_path(to_url.path) - if not (to_path + "/").startswith(base_prefix + "/"): - logger.warning("Destination %r from MOVE request on %r doesn't " - "start with base prefix", to_path, path) - return NOT_ALLOWED - to_path = to_path[len(base_prefix):] - if not self._access(user, to_path, "w"): - return NOT_ALLOWED - - with self.Collection.acquire_lock("w", user): - item = next(self.Collection.discover(path), None) - if not item: - return NOT_FOUND - if (not self._access(user, path, "w", item) or - not self._access(user, to_path, "w", item)): - return NOT_ALLOWED - if isinstance(item, storage.BaseCollection): - # TODO: support moving collections - return METHOD_NOT_ALLOWED - - to_item = next(self.Collection.discover(to_path), None) - if isinstance(to_item, storage.BaseCollection): - return FORBIDDEN - to_parent_path = storage.sanitize_path( - "/%s/" % posixpath.dirname(to_path.strip("/"))) - to_collection = next( - self.Collection.discover(to_parent_path), None) - if not to_collection: - return CONFLICT - tag = item.collection.get_meta("tag") - if not tag or tag != to_collection.get_meta("tag"): - return FORBIDDEN - if to_item and environ.get("HTTP_OVERWRITE", "F") != "T": - return PRECONDITION_FAILED - if (to_item and item.uid != to_item.uid or - not to_item and - to_collection.path != item.collection.path and - to_collection.has_uid(item.uid)): - return self._webdav_error_response( - "C" if tag == "VCALENDAR" else "CR", "no-uid-conflict") - to_href = posixpath.basename(to_path.strip("/")) - try: - self.Collection.move(item, to_collection, to_href) - except ValueError as e: - logger.warning( - "Bad MOVE request on %r: %s", path, e, exc_info=True) - return BAD_REQUEST - return client.NO_CONTENT if to_item else client.CREATED, {}, None - - def do_OPTIONS(self, environ, base_prefix, path, user): - """Manage OPTIONS request.""" - headers = { - "Allow": ", ".join( - name[3:] for name in dir(self) if name.startswith("do_")), - "DAV": DAV_HEADERS} - return client.OK, headers, None - - def do_PROPFIND(self, environ, base_prefix, path, user): - """Manage PROPFIND request.""" - if not self._access(user, path, "r"): - return NOT_ALLOWED - try: - xml_content = self._read_xml_content(environ) - except RuntimeError as e: - logger.warning( - "Bad PROPFIND request on %r: %s", path, e, exc_info=True) - return BAD_REQUEST - except socket.timeout as e: - logger.debug("client timed out", exc_info=True) - return REQUEST_TIMEOUT - with self.Collection.acquire_lock("r", user): - items = self.Collection.discover( - path, environ.get("HTTP_DEPTH", "0")) - # take root item for rights checking - item = next(items, None) - if not item: - return NOT_FOUND - if not self._access(user, path, "r", item): - return NOT_ALLOWED - # put item back - items = itertools.chain([item], items) - allowed_items = self.collect_allowed_items(items, user) - headers = {"DAV": DAV_HEADERS, - "Content-Type": "text/xml; charset=%s" % self.encoding} - status, xml_answer = xmlutils.propfind( - base_prefix, path, xml_content, allowed_items, user) - if status == client.FORBIDDEN: - return NOT_ALLOWED - return status, headers, self._write_xml_content(xml_answer) - - def do_PROPPATCH(self, environ, base_prefix, path, user): - """Manage PROPPATCH request.""" - if not self._access(user, path, "w"): - return NOT_ALLOWED - try: - xml_content = self._read_xml_content(environ) - except RuntimeError as e: - logger.warning( - "Bad PROPPATCH request on %r: %s", path, e, exc_info=True) - return BAD_REQUEST - except socket.timeout as e: - logger.debug("client timed out", exc_info=True) - return REQUEST_TIMEOUT - with self.Collection.acquire_lock("w", user): - item = next(self.Collection.discover(path), None) - if not item: - return NOT_FOUND - if not self._access(user, path, "w", item): - return NOT_ALLOWED - if not isinstance(item, storage.BaseCollection): - return FORBIDDEN - headers = {"DAV": DAV_HEADERS, - "Content-Type": "text/xml; charset=%s" % self.encoding} - try: - xml_answer = xmlutils.proppatch(base_prefix, path, xml_content, - item) - except ValueError as e: - logger.warning( - "Bad PROPPATCH request on %r: %s", path, e, exc_info=True) - return BAD_REQUEST - return (client.MULTI_STATUS, headers, - self._write_xml_content(xml_answer)) - - def do_PUT(self, environ, base_prefix, path, user): - """Manage PUT request.""" - if not self._access(user, path, "w"): - return NOT_ALLOWED - try: - content = self._read_content(environ) - except RuntimeError as e: - logger.warning("Bad PUT request on %r: %s", path, e, exc_info=True) - return BAD_REQUEST - except socket.timeout as e: - logger.debug("client timed out", exc_info=True) - return REQUEST_TIMEOUT - # Prepare before locking - parent_path = storage.sanitize_path( - "/%s/" % posixpath.dirname(path.strip("/"))) - permissions = self.Rights.authorized(user, path, "Ww") - parent_permissions = self.Rights.authorized(user, parent_path, "w") - - def prepare(vobject_items, tag=None, write_whole_collection=None): - if (write_whole_collection or - permissions and not parent_permissions): - write_whole_collection = True - content_type = environ.get("CONTENT_TYPE", - "").split(";")[0] - tags = {value: key - for key, value in xmlutils.MIMETYPES.items()} - tag = storage.predict_tag_of_whole_collection( - vobject_items, tags.get(content_type)) - if not tag: - raise ValueError("Can't determine collection tag") - collection_path = storage.sanitize_path(path).strip("/") - elif (write_whole_collection is not None and - not write_whole_collection or - not permissions and parent_permissions): - write_whole_collection = False - if tag is None: - tag = storage.predict_tag_of_parent_collection( - vobject_items) - collection_path = posixpath.dirname( - storage.sanitize_path(path).strip("/")) - props = None - stored_exc_info = None - items = [] - try: - if tag: - storage.check_and_sanitize_items( - vobject_items, is_collection=write_whole_collection, - tag=tag) - if write_whole_collection and tag == "VCALENDAR": - vobject_components = [] - vobject_item, = vobject_items - for content in ("vevent", "vtodo", "vjournal"): - vobject_components.extend( - getattr(vobject_item, "%s_list" % content, [])) - vobject_components_by_uid = itertools.groupby( - sorted(vobject_components, key=storage.get_uid), - storage.get_uid) - for uid, components in vobject_components_by_uid: - vobject_collection = vobject.iCalendar() - for component in components: - vobject_collection.add(component) - item = storage.Item( - collection_path=collection_path, - vobject_item=vobject_collection) - item.prepare() - items.append(item) - elif write_whole_collection and tag == "VADDRESSBOOK": - for vobject_item in vobject_items: - item = storage.Item( - collection_path=collection_path, - vobject_item=vobject_item) - item.prepare() - items.append(item) - elif not write_whole_collection: - vobject_item, = vobject_items - item = storage.Item(collection_path=collection_path, - vobject_item=vobject_item) - item.prepare() - items.append(item) - - if write_whole_collection: - props = {} - if tag: - props["tag"] = tag - if tag == "VCALENDAR" and vobject_items: - if hasattr(vobject_items[0], "x_wr_calname"): - calname = vobject_items[0].x_wr_calname.value - if calname: - props["D:displayname"] = calname - if hasattr(vobject_items[0], "x_wr_caldesc"): - caldesc = vobject_items[0].x_wr_caldesc.value - if caldesc: - props["C:calendar-description"] = caldesc - storage.check_and_sanitize_props(props) - except Exception: - stored_exc_info = sys.exc_info() - - # Use generator for items and delete references to free memory - # early - def items_generator(): - while items: - yield items.pop(0) - - return (items_generator(), tag, write_whole_collection, props, - stored_exc_info) - - try: - vobject_items = tuple(vobject.readComponents(content or "")) - except Exception as e: - logger.warning( - "Bad PUT request on %r: %s", path, e, exc_info=True) - return BAD_REQUEST - (prepared_items, prepared_tag, prepared_write_whole_collection, - prepared_props, prepared_exc_info) = prepare(vobject_items) - - with self.Collection.acquire_lock("w", user): - item = next(self.Collection.discover(path), None) - parent_item = next(self.Collection.discover(parent_path), None) - if not parent_item: - return CONFLICT - - write_whole_collection = ( - isinstance(item, storage.BaseCollection) or - not parent_item.get_meta("tag")) - - if write_whole_collection: - tag = prepared_tag - else: - tag = parent_item.get_meta("tag") - - if write_whole_collection: - if not self.Rights.authorized(user, path, "w" if tag else "W"): - return NOT_ALLOWED - elif not self.Rights.authorized(user, parent_path, "w"): - return NOT_ALLOWED - - etag = environ.get("HTTP_IF_MATCH", "") - if not item and etag: - # Etag asked but no item found: item has been removed - return PRECONDITION_FAILED - if item and etag and item.etag != etag: - # Etag asked but item not matching: item has changed - return PRECONDITION_FAILED - - match = environ.get("HTTP_IF_NONE_MATCH", "") == "*" - if item and match: - # Creation asked but item found: item can't be replaced - return PRECONDITION_FAILED - - if (tag != prepared_tag or - prepared_write_whole_collection != write_whole_collection): - (prepared_items, prepared_tag, prepared_write_whole_collection, - prepared_props, prepared_exc_info) = prepare( - vobject_items, tag, write_whole_collection) - props = prepared_props - if prepared_exc_info: - logger.warning( - "Bad PUT request on %r: %s", path, prepared_exc_info[1], - exc_info=prepared_exc_info) - return BAD_REQUEST - - if write_whole_collection: - try: - etag = self.Collection.create_collection( - path, prepared_items, props).etag - except ValueError as e: - logger.warning( - "Bad PUT request on %r: %s", path, e, exc_info=True) - return BAD_REQUEST - else: - prepared_item, = prepared_items - if (item and item.uid != prepared_item.uid or - not item and parent_item.has_uid(prepared_item.uid)): - return self._webdav_error_response( - "C" if tag == "VCALENDAR" else "CR", - "no-uid-conflict") - - href = posixpath.basename(path.strip("/")) - try: - etag = parent_item.upload(href, prepared_item).etag - except ValueError as e: - logger.warning( - "Bad PUT request on %r: %s", path, e, exc_info=True) - return BAD_REQUEST - - headers = {"ETag": etag} - return client.CREATED, headers, None - - def do_REPORT(self, environ, base_prefix, path, user): - """Manage REPORT request.""" - if not self._access(user, path, "r"): - return NOT_ALLOWED - try: - xml_content = self._read_xml_content(environ) - except RuntimeError as e: - logger.warning( - "Bad REPORT request on %r: %s", path, e, exc_info=True) - return BAD_REQUEST - except socket.timeout as e: - logger.debug("client timed out", exc_info=True) - return REQUEST_TIMEOUT - with contextlib.ExitStack() as lock_stack: - lock_stack.enter_context(self.Collection.acquire_lock("r", user)) - item = next(self.Collection.discover(path), None) - if not item: - return NOT_FOUND - if not self._access(user, path, "r", item): - return NOT_ALLOWED - if isinstance(item, storage.BaseCollection): - collection = item - else: - collection = item.collection - headers = {"Content-Type": "text/xml; charset=%s" % self.encoding} - try: - status, xml_answer = xmlutils.report( - base_prefix, path, xml_content, collection, - lock_stack.close) - except ValueError as e: - logger.warning( - "Bad REPORT request on %r: %s", path, e, exc_info=True) - return BAD_REQUEST - return (status, headers, self._write_xml_content(xml_answer)) - - _application = None _application_config_path = None _application_lock = threading.Lock() diff --git a/radicale/__main__.py b/radicale/__main__.py index 43654a91..cf89823c 100644 --- a/radicale/__main__.py +++ b/radicale/__main__.py @@ -1,5 +1,6 @@ # This file is part of Radicale Server - Calendar Server # Copyright © 2011-2017 Guillaume Ayoub +# Copyright © 2017-2018 Unrud # # This library is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by diff --git a/radicale/app/__init__.py b/radicale/app/__init__.py new file mode 100644 index 00000000..91fe68f7 --- /dev/null +++ b/radicale/app/__init__.py @@ -0,0 +1,376 @@ +# This file is part of Radicale Server - Calendar Server +# Copyright © 2008 Nicolas Kandel +# Copyright © 2008 Pascal Halter +# Copyright © 2008-2017 Guillaume Ayoub +# Copyright © 2017-2018 Unrud +# +# This library is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Radicale. If not, see . + +""" +Radicale WSGI application. + +Can be used with an external WSGI server or the built-in server. + +""" + +import base64 +import datetime +import io +import logging +import pkg_resources +import posixpath +import pprint +import random +import time +import zlib +from http import client +from xml.etree import ElementTree as ET + +from radicale import ( + auth, httputils, log, pathutils, rights, storage, web, xmlutils) +from radicale.app.delete import ApplicationDeleteMixin +from radicale.app.get import ApplicationGetMixin +from radicale.app.head import ApplicationHeadMixin +from radicale.app.mkcalendar import ApplicationMkcalendarMixin +from radicale.app.mkcol import ApplicationMkcolMixin +from radicale.app.move import ApplicationMoveMixin +from radicale.app.options import ApplicationOptionsMixin +from radicale.app.propfind import ApplicationPropfindMixin +from radicale.app.proppatch import ApplicationProppatchMixin +from radicale.app.put import ApplicationPutMixin +from radicale.app.report import ApplicationReportMixin +from radicale.log import logger + +VERSION = pkg_resources.get_distribution("radicale").version + + +class Application( + ApplicationDeleteMixin, ApplicationGetMixin, ApplicationHeadMixin, + ApplicationMkcalendarMixin, ApplicationMkcolMixin, + ApplicationMoveMixin, ApplicationOptionsMixin, + ApplicationPropfindMixin, ApplicationProppatchMixin, + ApplicationPutMixin, ApplicationReportMixin): + + """WSGI application managing collections.""" + + def __init__(self, configuration): + """Initialize application.""" + super().__init__() + self.configuration = configuration + self.Auth = auth.load(configuration) + self.Collection = storage.load(configuration) + self.Rights = rights.load(configuration) + self.Web = web.load(configuration) + self.encoding = configuration.get("encoding", "request") + + def _headers_log(self, environ): + """Sanitize headers for logging.""" + request_environ = dict(environ) + + # Mask passwords + mask_passwords = self.configuration.getboolean( + "logging", "mask_passwords") + authorization = request_environ.get("HTTP_AUTHORIZATION", "") + if mask_passwords and authorization.startswith("Basic"): + request_environ["HTTP_AUTHORIZATION"] = "Basic **masked**" + if request_environ.get("HTTP_COOKIE"): + request_environ["HTTP_COOKIE"] = "**masked**" + + return request_environ + + def decode(self, text, environ): + """Try to magically decode ``text`` according to given ``environ``.""" + # List of charsets to try + charsets = [] + + # First append content charset given in the request + content_type = environ.get("CONTENT_TYPE") + if content_type and "charset=" in content_type: + charsets.append( + content_type.split("charset=")[1].split(";")[0].strip()) + # Then append default Radicale charset + charsets.append(self.encoding) + # Then append various fallbacks + charsets.append("utf-8") + charsets.append("iso8859-1") + + # Try to decode + for charset in charsets: + try: + return text.decode(charset) + except UnicodeDecodeError: + pass + raise UnicodeDecodeError + + def __call__(self, environ, start_response): + with log.register_stream(environ["wsgi.errors"]): + try: + status, headers, answers = self._handle_request(environ) + except Exception as e: + try: + method = str(environ["REQUEST_METHOD"]) + except Exception: + method = "unknown" + try: + path = str(environ.get("PATH_INFO", "")) + except Exception: + path = "" + logger.error("An exception occurred during %s request on %r: " + "%s", method, path, e, exc_info=True) + status, headers, answer = httputils.INTERNAL_SERVER_ERROR + answer = answer.encode("ascii") + status = "%d %s" % ( + status, client.responses.get(status, "Unknown")) + headers = [ + ("Content-Length", str(len(answer)))] + list(headers) + answers = [answer] + start_response(status, headers) + return answers + + def _handle_request(self, environ): + """Manage a request.""" + def response(status, headers=(), answer=None): + headers = dict(headers) + # Set content length + if answer: + if hasattr(answer, "encode"): + logger.debug("Response content:\n%s", answer) + headers["Content-Type"] += "; charset=%s" % self.encoding + answer = answer.encode(self.encoding) + accept_encoding = [ + encoding.strip() for encoding in + environ.get("HTTP_ACCEPT_ENCODING", "").split(",") + if encoding.strip()] + + if "gzip" in accept_encoding: + zcomp = zlib.compressobj(wbits=16 + zlib.MAX_WBITS) + answer = zcomp.compress(answer) + zcomp.flush() + headers["Content-Encoding"] = "gzip" + + headers["Content-Length"] = str(len(answer)) + + # Add extra headers set in configuration + if self.configuration.has_section("headers"): + for key in self.configuration.options("headers"): + headers[key] = self.configuration.get("headers", key) + + # Start response + time_end = datetime.datetime.now() + status = "%d %s" % ( + status, client.responses.get(status, "Unknown")) + logger.info( + "%s response status for %r%s in %.3f seconds: %s", + environ["REQUEST_METHOD"], environ.get("PATH_INFO", ""), + depthinfo, (time_end - time_begin).total_seconds(), status) + # Return response content + return status, list(headers.items()), [answer] if answer else [] + + remote_host = "unknown" + if environ.get("REMOTE_HOST"): + remote_host = repr(environ["REMOTE_HOST"]) + elif environ.get("REMOTE_ADDR"): + remote_host = environ["REMOTE_ADDR"] + if environ.get("HTTP_X_FORWARDED_FOR"): + remote_host = "%r (forwarded by %s)" % ( + environ["HTTP_X_FORWARDED_FOR"], remote_host) + remote_useragent = "" + if environ.get("HTTP_USER_AGENT"): + remote_useragent = " using %r" % environ["HTTP_USER_AGENT"] + depthinfo = "" + if environ.get("HTTP_DEPTH"): + depthinfo = " with depth %r" % environ["HTTP_DEPTH"] + time_begin = datetime.datetime.now() + logger.info( + "%s request for %r%s received from %s%s", + environ["REQUEST_METHOD"], environ.get("PATH_INFO", ""), depthinfo, + remote_host, remote_useragent) + headers = pprint.pformat(self._headers_log(environ)) + logger.debug("Request headers:\n%s", headers) + + # Let reverse proxies overwrite SCRIPT_NAME + if "HTTP_X_SCRIPT_NAME" in environ: + # script_name must be removed from PATH_INFO by the client. + unsafe_base_prefix = environ["HTTP_X_SCRIPT_NAME"] + logger.debug("Script name overwritten by client: %r", + unsafe_base_prefix) + else: + # SCRIPT_NAME is already removed from PATH_INFO, according to the + # WSGI specification. + unsafe_base_prefix = environ.get("SCRIPT_NAME", "") + # Sanitize base prefix + base_prefix = pathutils.sanitize_path(unsafe_base_prefix).rstrip("/") + logger.debug("Sanitized script name: %r", base_prefix) + # Sanitize request URI (a WSGI server indicates with an empty path, + # that the URL targets the application root without a trailing slash) + path = pathutils.sanitize_path(environ.get("PATH_INFO", "")) + logger.debug("Sanitized path: %r", path) + + # Get function corresponding to method + function = getattr(self, "do_%s" % environ["REQUEST_METHOD"].upper()) + + # If "/.well-known" is not available, clients query "/" + if path == "/.well-known" or path.startswith("/.well-known/"): + return response(*httputils.NOT_FOUND) + + # Ask authentication backend to check rights + login = password = "" + external_login = self.Auth.get_external_login(environ) + authorization = environ.get("HTTP_AUTHORIZATION", "") + if external_login: + login, password = external_login + login, password = login or "", password or "" + elif authorization.startswith("Basic"): + authorization = authorization[len("Basic"):].strip() + login, password = self.decode(base64.b64decode( + authorization.encode("ascii")), environ).split(":", 1) + + user = self.Auth.login(login, password) or "" if login else "" + if user and login == user: + logger.info("Successful login: %r", user) + elif user: + logger.info("Successful login: %r -> %r", login, user) + elif login: + logger.info("Failed login attempt: %r", login) + # Random delay to avoid timing oracles and bruteforce attacks + delay = self.configuration.getfloat("auth", "delay") + if delay > 0: + random_delay = delay * (0.5 + random.random()) + logger.debug("Sleeping %.3f seconds", random_delay) + time.sleep(random_delay) + + if user and not pathutils.is_safe_path_component(user): + # Prevent usernames like "user/calendar.ics" + logger.info("Refused unsafe username: %r", user) + user = "" + + # Create principal collection + if user: + principal_path = "/%s/" % user + if self.Rights.authorized(user, principal_path, "W"): + with self.Collection.acquire_lock("r", user): + principal = next( + self.Collection.discover(principal_path, depth="1"), + None) + if not principal: + with self.Collection.acquire_lock("w", user): + try: + self.Collection.create_collection(principal_path) + except ValueError as e: + logger.warning("Failed to create principal " + "collection %r: %s", user, e) + user = "" + else: + logger.warning("Access to principal path %r denied by " + "rights backend", principal_path) + + if self.configuration.getboolean("internal", "internal_server"): + # Verify content length + content_length = int(environ.get("CONTENT_LENGTH") or 0) + if content_length: + max_content_length = self.configuration.getint( + "server", "max_content_length") + if max_content_length and content_length > max_content_length: + logger.info("Request body too large: %d", content_length) + return response(*httputils.REQUEST_ENTITY_TOO_LARGE) + + if not login or user: + status, headers, answer = function( + environ, base_prefix, path, user) + if (status, headers, answer) == httputils.NOT_ALLOWED: + logger.info("Access to %r denied for %s", path, + repr(user) if user else "anonymous user") + else: + status, headers, answer = httputils.NOT_ALLOWED + + if ((status, headers, answer) == httputils.NOT_ALLOWED and not user and + not external_login): + # Unknown or unauthorized user + logger.debug("Asking client for authentication") + status = client.UNAUTHORIZED + realm = self.configuration.get("auth", "realm") + headers = dict(headers) + headers.update({ + "WWW-Authenticate": + "Basic realm=\"%s\"" % realm}) + + return response(status, headers, answer) + + def access(self, user, path, permission, item=None): + if permission not in "rw": + raise ValueError("Invalid permission argument: %r" % permission) + if not item: + permissions = permission + permission.upper() + parent_permissions = permission + elif isinstance(item, storage.BaseCollection): + if item.get_meta("tag"): + permissions = permission + else: + permissions = permission.upper() + parent_permissions = "" + else: + permissions = "" + parent_permissions = permission + if permissions and self.Rights.authorized(user, path, permissions): + return True + if parent_permissions: + parent_path = pathutils.sanitize_path( + "/%s/" % posixpath.dirname(path.strip("/"))) + if self.Rights.authorized(user, parent_path, parent_permissions): + return True + return False + + def read_raw_content(self, environ): + content_length = int(environ.get("CONTENT_LENGTH") or 0) + if not content_length: + return b"" + content = environ["wsgi.input"].read(content_length) + if len(content) < content_length: + raise RuntimeError("Request body too short: %d" % len(content)) + return content + + def read_content(self, environ): + content = self.decode(self.read_raw_content(environ), environ) + logger.debug("Request content:\n%s", content) + return content + + def read_xml_content(self, environ): + content = self.decode(self.read_raw_content(environ), environ) + if not content: + return None + try: + xml_content = ET.fromstring(content) + except ET.ParseError as e: + logger.debug("Request content (Invalid XML):\n%s", content) + raise RuntimeError("Failed to parse XML: %s" % e) from e + if logger.isEnabledFor(logging.DEBUG): + logger.debug("Request content:\n%s", + xmlutils.pretty_xml(xml_content)) + return xml_content + + def write_xml_content(self, xml_content): + if logger.isEnabledFor(logging.DEBUG): + logger.debug("Response content:\n%s", + xmlutils.pretty_xml(xml_content)) + f = io.BytesIO() + ET.ElementTree(xml_content).write(f, encoding=self.encoding, + xml_declaration=True) + return f.getvalue() + + def webdav_error_response(self, namespace, name, + status=httputils.WEBDAV_PRECONDITION_FAILED[0]): + """Generate XML error response.""" + headers = {"Content-Type": "text/xml; charset=%s" % self.encoding} + content = self.write_xml_content( + xmlutils.webdav_error(namespace, name)) + return status, headers, content diff --git a/radicale/app/delete.py b/radicale/app/delete.py new file mode 100644 index 00000000..4a60a964 --- /dev/null +++ b/radicale/app/delete.py @@ -0,0 +1,70 @@ +# This file is part of Radicale Server - Calendar Server +# Copyright © 2008 Nicolas Kandel +# Copyright © 2008 Pascal Halter +# Copyright © 2008-2017 Guillaume Ayoub +# Copyright © 2017-2018 Unrud +# +# This library is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Radicale. If not, see . + +from http import client +from xml.etree import ElementTree as ET + +from radicale import httputils, storage, xmlutils + + +def xml_delete(base_prefix, path, collection, href=None): + """Read and answer DELETE requests. + + Read rfc4918-9.6 for info. + + """ + collection.delete(href) + + multistatus = ET.Element(xmlutils.make_tag("D", "multistatus")) + response = ET.Element(xmlutils.make_tag("D", "response")) + multistatus.append(response) + + href = ET.Element(xmlutils.make_tag("D", "href")) + href.text = xmlutils.make_href(base_prefix, path) + response.append(href) + + status = ET.Element(xmlutils.make_tag("D", "status")) + status.text = xmlutils.make_response(200) + response.append(status) + + return multistatus + + +class ApplicationDeleteMixin: + def do_DELETE(self, environ, base_prefix, path, user): + """Manage DELETE request.""" + if not self.access(user, path, "w"): + return httputils.NOT_ALLOWED + with self.Collection.acquire_lock("w", user): + item = next(self.Collection.discover(path), None) + if not item: + return httputils.NOT_FOUND + if not self.access(user, path, "w", item): + return httputils.NOT_ALLOWED + if_match = environ.get("HTTP_IF_MATCH", "*") + if if_match not in ("*", item.etag): + # ETag precondition not verified, do not delete item + return httputils.PRECONDITION_FAILED + if isinstance(item, storage.BaseCollection): + xml_answer = xml_delete(base_prefix, path, item) + else: + xml_answer = xml_delete( + base_prefix, path, item.collection, item.href) + headers = {"Content-Type": "text/xml; charset=%s" % self.encoding} + return client.OK, headers, self.write_xml_content(xml_answer) diff --git a/radicale/app/get.py b/radicale/app/get.py new file mode 100644 index 00000000..b98bbd40 --- /dev/null +++ b/radicale/app/get.py @@ -0,0 +1,105 @@ +# This file is part of Radicale Server - Calendar Server +# Copyright © 2008 Nicolas Kandel +# Copyright © 2008 Pascal Halter +# Copyright © 2008-2017 Guillaume Ayoub +# Copyright © 2017-2018 Unrud +# +# This library is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Radicale. If not, see . + +""" +Radicale WSGI application. + +Can be used with an external WSGI server or the built-in server. + +""" + +import posixpath +from http import client +from urllib.parse import quote + +from radicale import httputils, storage, xmlutils +from radicale.log import logger + + +def propose_filename(collection): + """Propose a filename for a collection.""" + tag = collection.get_meta("tag") + if tag == "VADDRESSBOOK": + fallback_title = "Address book" + suffix = ".vcf" + elif tag == "VCALENDAR": + fallback_title = "Calendar" + suffix = ".ics" + else: + fallback_title = posixpath.basename(collection.path) + suffix = "" + title = collection.get_meta("D:displayname") or fallback_title + if title and not title.lower().endswith(suffix.lower()): + title += suffix + return title + + +class ApplicationGetMixin: + def _content_disposition_attachement(self, filename): + value = "attachement" + try: + encoded_filename = quote(filename, encoding=self.encoding) + except UnicodeEncodeError as e: + logger.warning("Failed to encode filename: %r", filename, + exc_info=True) + encoded_filename = "" + if encoded_filename: + value += "; filename*=%s''%s" % (self.encoding, encoded_filename) + return value + + def do_GET(self, environ, base_prefix, path, user): + """Manage GET request.""" + # Redirect to .web if the root URL is requested + if not path.strip("/"): + web_path = ".web" + if not environ.get("PATH_INFO"): + web_path = posixpath.join(posixpath.basename(base_prefix), + web_path) + return (client.FOUND, + {"Location": web_path, "Content-Type": "text/plain"}, + "Redirected to %s" % web_path) + # Dispatch .web URL to web module + if path == "/.web" or path.startswith("/.web/"): + return self.Web.get(environ, base_prefix, path, user) + if not self.access(user, path, "r"): + return httputils.NOT_ALLOWED + with self.Collection.acquire_lock("r", user): + item = next(self.Collection.discover(path), None) + if not item: + return httputils.NOT_FOUND + if not self.access(user, path, "r", item): + return httputils.NOT_ALLOWED + if isinstance(item, storage.BaseCollection): + tag = item.get_meta("tag") + if not tag: + return httputils.DIRECTORY_LISTING + content_type = xmlutils.MIMETYPES[tag] + content_disposition = self._content_disposition_attachement( + propose_filename(item)) + else: + content_type = xmlutils.OBJECT_MIMETYPES[item.name] + content_disposition = "" + headers = { + "Content-Type": content_type, + "Last-Modified": item.last_modified, + "ETag": item.etag} + if content_disposition: + headers["Content-Disposition"] = content_disposition + answer = item.serialize() + return client.OK, headers, answer diff --git a/radicale/app/head.py b/radicale/app/head.py new file mode 100644 index 00000000..1b434ecd --- /dev/null +++ b/radicale/app/head.py @@ -0,0 +1,33 @@ +# This file is part of Radicale Server - Calendar Server +# Copyright © 2008 Nicolas Kandel +# Copyright © 2008 Pascal Halter +# Copyright © 2008-2017 Guillaume Ayoub +# Copyright © 2017-2018 Unrud +# +# This library is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Radicale. If not, see . + +""" +Radicale WSGI application. + +Can be used with an external WSGI server or the built-in server. + +""" + + +class ApplicationHeadMixin: + def do_HEAD(self, environ, base_prefix, path, user): + """Manage HEAD request.""" + status, headers, answer = self.do_GET( + environ, base_prefix, path, user) + return status, headers, None diff --git a/radicale/app/mkcalendar.py b/radicale/app/mkcalendar.py new file mode 100644 index 00000000..0fa0ad7e --- /dev/null +++ b/radicale/app/mkcalendar.py @@ -0,0 +1,80 @@ +# This file is part of Radicale Server - Calendar Server +# Copyright © 2008 Nicolas Kandel +# Copyright © 2008 Pascal Halter +# Copyright © 2008-2017 Guillaume Ayoub +# Copyright © 2017-2018 Unrud +# +# This library is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Radicale. If not, see . + +""" +Radicale WSGI application. + +Can be used with an external WSGI server or the built-in server. + +""" + +import posixpath +import socket +from http import client + +from radicale import httputils +from radicale import item as radicale_item +from radicale import pathutils, storage, xmlutils +from radicale.log import logger + + +class ApplicationMkcalendarMixin: + def do_MKCALENDAR(self, environ, base_prefix, path, user): + """Manage MKCALENDAR request.""" + if not self.Rights.authorized(user, path, "w"): + return httputils.NOT_ALLOWED + try: + xml_content = self.read_xml_content(environ) + except RuntimeError as e: + logger.warning( + "Bad MKCALENDAR request on %r: %s", path, e, exc_info=True) + return httputils.BAD_REQUEST + except socket.timeout as e: + logger.debug("client timed out", exc_info=True) + return httputils.REQUEST_TIMEOUT + # Prepare before locking + props = xmlutils.props_from_request(xml_content) + props["tag"] = "VCALENDAR" + # TODO: use this? + # timezone = props.get("C:calendar-timezone") + try: + radicale_item.check_and_sanitize_props(props) + except ValueError as e: + logger.warning( + "Bad MKCALENDAR request on %r: %s", path, e, exc_info=True) + with self.Collection.acquire_lock("w", user): + item = next(self.Collection.discover(path), None) + if item: + return self.webdav_error_response( + "D", "resource-must-be-null") + parent_path = pathutils.sanitize_path( + "/%s/" % posixpath.dirname(path.strip("/"))) + parent_item = next(self.Collection.discover(parent_path), None) + if not parent_item: + return httputils.CONFLICT + if (not isinstance(parent_item, storage.BaseCollection) or + parent_item.get_meta("tag")): + return httputils.FORBIDDEN + try: + self.Collection.create_collection(path, props=props) + except ValueError as e: + logger.warning( + "Bad MKCALENDAR request on %r: %s", path, e, exc_info=True) + return httputils.BAD_REQUEST + return client.CREATED, {}, None diff --git a/radicale/app/mkcol.py b/radicale/app/mkcol.py new file mode 100644 index 00000000..89dc7c00 --- /dev/null +++ b/radicale/app/mkcol.py @@ -0,0 +1,81 @@ +# This file is part of Radicale Server - Calendar Server +# Copyright © 2008 Nicolas Kandel +# Copyright © 2008 Pascal Halter +# Copyright © 2008-2017 Guillaume Ayoub +# Copyright © 2017-2018 Unrud +# +# This library is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Radicale. If not, see . + +""" +Radicale WSGI application. + +Can be used with an external WSGI server or the built-in server. + +""" + +import posixpath +import socket +from http import client + +from radicale import httputils +from radicale import item as radicale_item +from radicale import pathutils, storage, xmlutils +from radicale.log import logger + + +class ApplicationMkcolMixin: + def do_MKCOL(self, environ, base_prefix, path, user): + """Manage MKCOL request.""" + permissions = self.Rights.authorized(user, path, "Ww") + if not permissions: + return httputils.NOT_ALLOWED + try: + xml_content = self.read_xml_content(environ) + except RuntimeError as e: + logger.warning( + "Bad MKCOL request on %r: %s", path, e, exc_info=True) + return httputils.BAD_REQUEST + except socket.timeout as e: + logger.debug("client timed out", exc_info=True) + return httputils.REQUEST_TIMEOUT + # Prepare before locking + props = xmlutils.props_from_request(xml_content) + try: + radicale_item.check_and_sanitize_props(props) + except ValueError as e: + logger.warning( + "Bad MKCOL request on %r: %s", path, e, exc_info=True) + return httputils.BAD_REQUEST + if (props.get("tag") and "w" not in permissions or + not props.get("tag") and "W" not in permissions): + return httputils.NOT_ALLOWED + with self.Collection.acquire_lock("w", user): + item = next(self.Collection.discover(path), None) + if item: + return httputils.METHOD_NOT_ALLOWED + parent_path = pathutils.sanitize_path( + "/%s/" % posixpath.dirname(path.strip("/"))) + parent_item = next(self.Collection.discover(parent_path), None) + if not parent_item: + return httputils.CONFLICT + if (not isinstance(parent_item, storage.BaseCollection) or + parent_item.get_meta("tag")): + return httputils.FORBIDDEN + try: + self.Collection.create_collection(path, props=props) + except ValueError as e: + logger.warning( + "Bad MKCOL request on %r: %s", path, e, exc_info=True) + return httputils.BAD_REQUEST + return client.CREATED, {}, None diff --git a/radicale/app/move.py b/radicale/app/move.py new file mode 100644 index 00000000..09295d7b --- /dev/null +++ b/radicale/app/move.py @@ -0,0 +1,93 @@ +# This file is part of Radicale Server - Calendar Server +# Copyright © 2008 Nicolas Kandel +# Copyright © 2008 Pascal Halter +# Copyright © 2008-2017 Guillaume Ayoub +# Copyright © 2017-2018 Unrud +# +# This library is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Radicale. If not, see . + +""" +Radicale WSGI application. + +Can be used with an external WSGI server or the built-in server. + +""" + +import posixpath +from http import client +from urllib.parse import urlparse + +from radicale import httputils, pathutils, storage +from radicale.log import logger + + +class ApplicationMoveMixin: + def do_MOVE(self, environ, base_prefix, path, user): + """Manage MOVE request.""" + raw_dest = environ.get("HTTP_DESTINATION", "") + to_url = urlparse(raw_dest) + if to_url.netloc != environ["HTTP_HOST"]: + logger.info("Unsupported destination address: %r", raw_dest) + # Remote destination server, not supported + return httputils.REMOTE_DESTINATION + if not self.access(user, path, "w"): + return httputils.NOT_ALLOWED + to_path = pathutils.sanitize_path(to_url.path) + if not (to_path + "/").startswith(base_prefix + "/"): + logger.warning("Destination %r from MOVE request on %r doesn't " + "start with base prefix", to_path, path) + return httputils.NOT_ALLOWED + to_path = to_path[len(base_prefix):] + if not self.access(user, to_path, "w"): + return httputils.NOT_ALLOWED + + with self.Collection.acquire_lock("w", user): + item = next(self.Collection.discover(path), None) + if not item: + return httputils.NOT_FOUND + if (not self.access(user, path, "w", item) or + not self.access(user, to_path, "w", item)): + return httputils.NOT_ALLOWED + if isinstance(item, storage.BaseCollection): + # TODO: support moving collections + return httputils.METHOD_NOT_ALLOWED + + to_item = next(self.Collection.discover(to_path), None) + if isinstance(to_item, storage.BaseCollection): + return httputils.FORBIDDEN + to_parent_path = pathutils.sanitize_path( + "/%s/" % posixpath.dirname(to_path.strip("/"))) + to_collection = next( + self.Collection.discover(to_parent_path), None) + if not to_collection: + return httputils.CONFLICT + tag = item.collection.get_meta("tag") + if not tag or tag != to_collection.get_meta("tag"): + return httputils.FORBIDDEN + if to_item and environ.get("HTTP_OVERWRITE", "F") != "T": + return httputils.PRECONDITION_FAILED + if (to_item and item.uid != to_item.uid or + not to_item and + to_collection.path != item.collection.path and + to_collection.has_uid(item.uid)): + return self.webdav_error_response( + "C" if tag == "VCALENDAR" else "CR", "no-uid-conflict") + to_href = posixpath.basename(to_path.strip("/")) + try: + self.Collection.move(item, to_collection, to_href) + except ValueError as e: + logger.warning( + "Bad MOVE request on %r: %s", path, e, exc_info=True) + return httputils.BAD_REQUEST + return client.NO_CONTENT if to_item else client.CREATED, {}, None diff --git a/radicale/app/options.py b/radicale/app/options.py new file mode 100644 index 00000000..34fb468d --- /dev/null +++ b/radicale/app/options.py @@ -0,0 +1,39 @@ +# This file is part of Radicale Server - Calendar Server +# Copyright © 2008 Nicolas Kandel +# Copyright © 2008 Pascal Halter +# Copyright © 2008-2017 Guillaume Ayoub +# Copyright © 2017-2018 Unrud +# +# This library is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Radicale. If not, see . + +""" +Radicale WSGI application. + +Can be used with an external WSGI server or the built-in server. + +""" + +from http import client + +from radicale import httputils + + +class ApplicationOptionsMixin: + def do_OPTIONS(self, environ, base_prefix, path, user): + """Manage OPTIONS request.""" + headers = { + "Allow": ", ".join( + name[3:] for name in dir(self) if name.startswith("do_")), + "DAV": httputils.DAV_HEADERS} + return client.OK, headers, None diff --git a/radicale/app/propfind.py b/radicale/app/propfind.py new file mode 100644 index 00000000..89871a55 --- /dev/null +++ b/radicale/app/propfind.py @@ -0,0 +1,395 @@ +# This file is part of Radicale Server - Calendar Server +# Copyright © 2008 Nicolas Kandel +# Copyright © 2008 Pascal Halter +# Copyright © 2008-2017 Guillaume Ayoub +# Copyright © 2017-2018 Unrud +# +# This library is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Radicale. If not, see . + +""" +Radicale WSGI application. + +Can be used with an external WSGI server or the built-in server. + +""" + +import itertools +import posixpath +import socket +from http import client +from xml.etree import ElementTree as ET + +from radicale import httputils, pathutils, rights, storage, xmlutils +from radicale.log import logger + + +def xml_propfind(base_prefix, path, xml_request, allowed_items, user): + """Read and answer PROPFIND requests. + + Read rfc4918-9.1 for info. + + The collections parameter is a list of collections that are to be included + in the output. + + """ + # A client may choose not to submit a request body. An empty PROPFIND + # request body MUST be treated as if it were an 'allprop' request. + top_tag = (xml_request[0] if xml_request is not None else + ET.Element(xmlutils.make_tag("D", "allprop"))) + + props = () + allprop = False + propname = False + if top_tag.tag == xmlutils.make_tag("D", "allprop"): + allprop = True + elif top_tag.tag == xmlutils.make_tag("D", "propname"): + propname = True + elif top_tag.tag == xmlutils.make_tag("D", "prop"): + props = [prop.tag for prop in top_tag] + + if xmlutils.make_tag("D", "current-user-principal") in props and not user: + # Ask for authentication + # Returning the DAV:unauthenticated pseudo-principal as specified in + # RFC 5397 doesn't seem to work with DAVdroid. + return client.FORBIDDEN, None + + # Writing answer + multistatus = ET.Element(xmlutils.make_tag("D", "multistatus")) + + for item, permission in allowed_items: + write = permission == "w" + response = xml_propfind_response( + base_prefix, path, item, props, user, write=write, + allprop=allprop, propname=propname) + if response: + multistatus.append(response) + + return client.MULTI_STATUS, multistatus + + +def xml_propfind_response(base_prefix, path, item, props, user, write=False, + propname=False, allprop=False): + """Build and return a PROPFIND response.""" + if propname and allprop or (props and (propname or allprop)): + raise ValueError("Only use one of props, propname and allprops") + is_collection = isinstance(item, storage.BaseCollection) + if is_collection: + is_leaf = item.get_meta("tag") in ("VADDRESSBOOK", "VCALENDAR") + collection = item + else: + collection = item.collection + + response = ET.Element(xmlutils.make_tag("D", "response")) + + href = ET.Element(xmlutils.make_tag("D", "href")) + if is_collection: + # Some clients expect collections to end with / + uri = "/%s/" % item.path if item.path else "/" + else: + uri = "/" + posixpath.join(collection.path, item.href) + + href.text = xmlutils.make_href(base_prefix, uri) + response.append(href) + + propstat404 = ET.Element(xmlutils.make_tag("D", "propstat")) + propstat200 = ET.Element(xmlutils.make_tag("D", "propstat")) + response.append(propstat200) + + prop200 = ET.Element(xmlutils.make_tag("D", "prop")) + propstat200.append(prop200) + + prop404 = ET.Element(xmlutils.make_tag("D", "prop")) + propstat404.append(prop404) + + if propname or allprop: + props = [] + # Should list all properties that can be retrieved by the code below + props.append(xmlutils.make_tag("D", "principal-collection-set")) + props.append(xmlutils.make_tag("D", "current-user-principal")) + props.append(xmlutils.make_tag("D", "current-user-privilege-set")) + props.append(xmlutils.make_tag("D", "supported-report-set")) + props.append(xmlutils.make_tag("D", "resourcetype")) + props.append(xmlutils.make_tag("D", "owner")) + + if is_collection and collection.is_principal: + props.append(xmlutils.make_tag("C", "calendar-user-address-set")) + props.append(xmlutils.make_tag("D", "principal-URL")) + props.append(xmlutils.make_tag("CR", "addressbook-home-set")) + props.append(xmlutils.make_tag("C", "calendar-home-set")) + + if not is_collection or is_leaf: + props.append(xmlutils.make_tag("D", "getetag")) + props.append(xmlutils.make_tag("D", "getlastmodified")) + props.append(xmlutils.make_tag("D", "getcontenttype")) + props.append(xmlutils.make_tag("D", "getcontentlength")) + + if is_collection: + if is_leaf: + props.append(xmlutils.make_tag("D", "displayname")) + props.append(xmlutils.make_tag("D", "sync-token")) + if collection.get_meta("tag") == "VCALENDAR": + props.append(xmlutils.make_tag("CS", "getctag")) + props.append( + xmlutils.make_tag("C", "supported-calendar-component-set")) + + meta = item.get_meta() + for tag in meta: + if tag == "tag": + continue + clark_tag = xmlutils.tag_from_human(tag) + if clark_tag not in props: + props.append(clark_tag) + + if propname: + for tag in props: + prop200.append(ET.Element(tag)) + props = () + + for tag in props: + element = ET.Element(tag) + is404 = False + if tag == xmlutils.make_tag("D", "getetag"): + if not is_collection or is_leaf: + element.text = item.etag + else: + is404 = True + elif tag == xmlutils.make_tag("D", "getlastmodified"): + if not is_collection or is_leaf: + element.text = item.last_modified + else: + is404 = True + elif tag == xmlutils.make_tag("D", "principal-collection-set"): + tag = ET.Element(xmlutils.make_tag("D", "href")) + tag.text = xmlutils.make_href(base_prefix, "/") + element.append(tag) + elif (tag in (xmlutils.make_tag("C", "calendar-user-address-set"), + xmlutils.make_tag("D", "principal-URL"), + xmlutils.make_tag("CR", "addressbook-home-set"), + xmlutils.make_tag("C", "calendar-home-set")) and + collection.is_principal and is_collection): + tag = ET.Element(xmlutils.make_tag("D", "href")) + tag.text = xmlutils.make_href(base_prefix, path) + element.append(tag) + elif tag == xmlutils.make_tag("C", "supported-calendar-component-set"): + human_tag = xmlutils.tag_from_clark(tag) + if is_collection and is_leaf: + meta = item.get_meta(human_tag) + if meta: + components = meta.split(",") + else: + components = ("VTODO", "VEVENT", "VJOURNAL") + for component in components: + comp = ET.Element(xmlutils.make_tag("C", "comp")) + comp.set("name", component) + element.append(comp) + else: + is404 = True + elif tag == xmlutils.make_tag("D", "current-user-principal"): + if user: + tag = ET.Element(xmlutils.make_tag("D", "href")) + tag.text = xmlutils.make_href(base_prefix, "/%s/" % user) + element.append(tag) + else: + element.append(ET.Element( + xmlutils.make_tag("D", "unauthenticated"))) + elif tag == xmlutils.make_tag("D", "current-user-privilege-set"): + privileges = [("D", "read")] + if write: + privileges.append(("D", "all")) + privileges.append(("D", "write")) + privileges.append(("D", "write-properties")) + privileges.append(("D", "write-content")) + for ns, privilege_name in privileges: + privilege = ET.Element(xmlutils.make_tag("D", "privilege")) + privilege.append(ET.Element( + xmlutils.make_tag(ns, privilege_name))) + element.append(privilege) + elif tag == xmlutils.make_tag("D", "supported-report-set"): + # These 3 reports are not implemented + reports = [ + ("D", "expand-property"), + ("D", "principal-search-property-set"), + ("D", "principal-property-search")] + if is_collection and is_leaf: + reports.append(("D", "sync-collection")) + if item.get_meta("tag") == "VADDRESSBOOK": + reports.append(("CR", "addressbook-multiget")) + reports.append(("CR", "addressbook-query")) + elif item.get_meta("tag") == "VCALENDAR": + reports.append(("C", "calendar-multiget")) + reports.append(("C", "calendar-query")) + for ns, report_name in reports: + supported = ET.Element( + xmlutils.make_tag("D", "supported-report")) + report_tag = ET.Element(xmlutils.make_tag("D", "report")) + supported_report_tag = ET.Element( + xmlutils.make_tag(ns, report_name)) + report_tag.append(supported_report_tag) + supported.append(report_tag) + element.append(supported) + elif tag == xmlutils.make_tag("D", "getcontentlength"): + if not is_collection or is_leaf: + encoding = collection.configuration.get("encoding", "request") + element.text = str(len(item.serialize().encode(encoding))) + else: + is404 = True + elif tag == xmlutils.make_tag("D", "owner"): + # return empty elment, if no owner available (rfc3744-5.1) + if collection.owner: + tag = ET.Element(xmlutils.make_tag("D", "href")) + tag.text = xmlutils.make_href( + base_prefix, "/%s/" % collection.owner) + element.append(tag) + elif is_collection: + if tag == xmlutils.make_tag("D", "getcontenttype"): + if is_leaf: + element.text = xmlutils.MIMETYPES[item.get_meta("tag")] + else: + is404 = True + elif tag == xmlutils.make_tag("D", "resourcetype"): + if item.is_principal: + tag = ET.Element(xmlutils.make_tag("D", "principal")) + element.append(tag) + if is_leaf: + if item.get_meta("tag") == "VADDRESSBOOK": + tag = ET.Element( + xmlutils.make_tag("CR", "addressbook")) + element.append(tag) + elif item.get_meta("tag") == "VCALENDAR": + tag = ET.Element(xmlutils.make_tag("C", "calendar")) + element.append(tag) + tag = ET.Element(xmlutils.make_tag("D", "collection")) + element.append(tag) + elif tag == xmlutils.make_tag("RADICALE", "displayname"): + # Only for internal use by the web interface + displayname = item.get_meta("D:displayname") + if displayname is not None: + element.text = displayname + else: + is404 = True + elif tag == xmlutils.make_tag("D", "displayname"): + displayname = item.get_meta("D:displayname") + if not displayname and is_leaf: + displayname = item.path + if displayname is not None: + element.text = displayname + else: + is404 = True + elif tag == xmlutils.make_tag("CS", "getctag"): + if is_leaf: + element.text = item.etag + else: + is404 = True + elif tag == xmlutils.make_tag("D", "sync-token"): + if is_leaf: + element.text, _ = item.sync() + else: + is404 = True + else: + human_tag = xmlutils.tag_from_clark(tag) + meta = item.get_meta(human_tag) + if meta is not None: + element.text = meta + else: + is404 = True + # Not for collections + elif tag == xmlutils.make_tag("D", "getcontenttype"): + element.text = xmlutils.get_content_type(item) + elif tag == xmlutils.make_tag("D", "resourcetype"): + # resourcetype must be returned empty for non-collection elements + pass + else: + is404 = True + + if is404: + prop404.append(element) + else: + prop200.append(element) + + status200 = ET.Element(xmlutils.make_tag("D", "status")) + status200.text = xmlutils.make_response(200) + propstat200.append(status200) + + status404 = ET.Element(xmlutils.make_tag("D", "status")) + status404.text = xmlutils.make_response(404) + propstat404.append(status404) + if len(prop404): + response.append(propstat404) + + return response + + +class ApplicationPropfindMixin: + def _collect_allowed_items(self, items, user): + """Get items from request that user is allowed to access.""" + for item in items: + if isinstance(item, storage.BaseCollection): + path = pathutils.sanitize_path("/%s/" % item.path) + if item.get_meta("tag"): + permissions = self.Rights.authorized(user, path, "rw") + target = "collection with tag %r" % item.path + else: + permissions = self.Rights.authorized(user, path, "RW") + target = "collection %r" % item.path + else: + path = pathutils.sanitize_path("/%s/" % item.collection.path) + permissions = self.Rights.authorized(user, path, "rw") + target = "item %r from %r" % (item.href, item.collection.path) + if rights.intersect_permissions(permissions, "Ww"): + permission = "w" + status = "write" + elif rights.intersect_permissions(permissions, "Rr"): + permission = "r" + status = "read" + else: + permission = "" + status = "NO" + logger.debug( + "%s has %s access to %s", + repr(user) if user else "anonymous user", status, target) + if permission: + yield item, permission + + def do_PROPFIND(self, environ, base_prefix, path, user): + """Manage PROPFIND request.""" + if not self.access(user, path, "r"): + return httputils.NOT_ALLOWED + try: + xml_content = self.read_xml_content(environ) + except RuntimeError as e: + logger.warning( + "Bad PROPFIND request on %r: %s", path, e, exc_info=True) + return httputils.BAD_REQUEST + except socket.timeout as e: + logger.debug("client timed out", exc_info=True) + return httputils.REQUEST_TIMEOUT + with self.Collection.acquire_lock("r", user): + items = self.Collection.discover( + path, environ.get("HTTP_DEPTH", "0")) + # take root item for rights checking + item = next(items, None) + if not item: + return httputils.NOT_FOUND + if not self.access(user, path, "r", item): + return httputils.NOT_ALLOWED + # put item back + items = itertools.chain([item], items) + allowed_items = self._collect_allowed_items(items, user) + headers = {"DAV": httputils.DAV_HEADERS, + "Content-Type": "text/xml; charset=%s" % self.encoding} + status, xml_answer = xml_propfind( + base_prefix, path, xml_content, allowed_items, user) + if status == client.FORBIDDEN: + return httputils.NOT_ALLOWED + return status, headers, self.write_xml_content(xml_answer) diff --git a/radicale/app/proppatch.py b/radicale/app/proppatch.py new file mode 100644 index 00000000..9206f4a5 --- /dev/null +++ b/radicale/app/proppatch.py @@ -0,0 +1,126 @@ +# This file is part of Radicale Server - Calendar Server +# Copyright © 2008 Nicolas Kandel +# Copyright © 2008 Pascal Halter +# Copyright © 2008-2017 Guillaume Ayoub +# Copyright © 2017-2018 Unrud +# +# This library is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Radicale. If not, see . + +""" +Radicale WSGI application. + +Can be used with an external WSGI server or the built-in server. + +""" + +import socket +from http import client +from xml.etree import ElementTree as ET + +from radicale import httputils +from radicale import item as radicale_item +from radicale import storage, xmlutils +from radicale.log import logger + + +def xml_add_propstat_to(element, tag, status_number): + """Add a PROPSTAT response structure to an element. + + The PROPSTAT answer structure is defined in rfc4918-9.1. It is added to the + given ``element``, for the following ``tag`` with the given + ``status_number``. + + """ + propstat = ET.Element(xmlutils.make_tag("D", "propstat")) + element.append(propstat) + + prop = ET.Element(xmlutils.make_tag("D", "prop")) + propstat.append(prop) + + clark_tag = tag if "{" in tag else xmlutils.make_tag(*tag.split(":", 1)) + prop_tag = ET.Element(clark_tag) + prop.append(prop_tag) + + status = ET.Element(xmlutils.make_tag("D", "status")) + status.text = xmlutils.make_response(status_number) + propstat.append(status) + + +def xml_proppatch(base_prefix, path, xml_request, collection): + """Read and answer PROPPATCH requests. + + Read rfc4918-9.2 for info. + + """ + props_to_set = xmlutils.props_from_request(xml_request, actions=("set",)) + props_to_remove = xmlutils.props_from_request(xml_request, + actions=("remove",)) + + multistatus = ET.Element(xmlutils.make_tag("D", "multistatus")) + response = ET.Element(xmlutils.make_tag("D", "response")) + multistatus.append(response) + + href = ET.Element(xmlutils.make_tag("D", "href")) + href.text = xmlutils.make_href(base_prefix, path) + response.append(href) + + new_props = collection.get_meta() + for short_name, value in props_to_set.items(): + new_props[short_name] = value + xml_add_propstat_to(response, short_name, 200) + for short_name in props_to_remove: + try: + del new_props[short_name] + except KeyError: + pass + xml_add_propstat_to(response, short_name, 200) + radicale_item.check_and_sanitize_props(new_props) + collection.set_meta(new_props) + + return multistatus + + +class ApplicationProppatchMixin: + def do_PROPPATCH(self, environ, base_prefix, path, user): + """Manage PROPPATCH request.""" + if not self.access(user, path, "w"): + return httputils.NOT_ALLOWED + try: + xml_content = self.read_xml_content(environ) + except RuntimeError as e: + logger.warning( + "Bad PROPPATCH request on %r: %s", path, e, exc_info=True) + return httputils.BAD_REQUEST + except socket.timeout as e: + logger.debug("client timed out", exc_info=True) + return httputils.REQUEST_TIMEOUT + with self.Collection.acquire_lock("w", user): + item = next(self.Collection.discover(path), None) + if not item: + return httputils.NOT_FOUND + if not self.access(user, path, "w", item): + return httputils.NOT_ALLOWED + if not isinstance(item, storage.BaseCollection): + return httputils.FORBIDDEN + headers = {"DAV": httputils.DAV_HEADERS, + "Content-Type": "text/xml; charset=%s" % self.encoding} + try: + xml_answer = xml_proppatch(base_prefix, path, xml_content, + item) + except ValueError as e: + logger.warning( + "Bad PROPPATCH request on %r: %s", path, e, exc_info=True) + return httputils.BAD_REQUEST + return (client.MULTI_STATUS, headers, + self.write_xml_content(xml_answer)) diff --git a/radicale/app/put.py b/radicale/app/put.py new file mode 100644 index 00000000..e10183fc --- /dev/null +++ b/radicale/app/put.py @@ -0,0 +1,230 @@ +# This file is part of Radicale Server - Calendar Server +# Copyright © 2008 Nicolas Kandel +# Copyright © 2008 Pascal Halter +# Copyright © 2008-2017 Guillaume Ayoub +# Copyright © 2017-2018 Unrud +# +# This library is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Radicale. If not, see . + +""" +Radicale WSGI application. + +Can be used with an external WSGI server or the built-in server. + +""" + +import itertools +import posixpath +import socket +import sys +from http import client + +import vobject + +from radicale import httputils +from radicale import item as radicale_item +from radicale import pathutils, storage, xmlutils +from radicale.log import logger + + +class ApplicationPutMixin: + def do_PUT(self, environ, base_prefix, path, user): + """Manage PUT request.""" + if not self.access(user, path, "w"): + return httputils.NOT_ALLOWED + try: + content = self.read_content(environ) + except RuntimeError as e: + logger.warning("Bad PUT request on %r: %s", path, e, exc_info=True) + return httputils.BAD_REQUEST + except socket.timeout as e: + logger.debug("client timed out", exc_info=True) + return httputils.REQUEST_TIMEOUT + # Prepare before locking + parent_path = pathutils.sanitize_path( + "/%s/" % posixpath.dirname(path.strip("/"))) + permissions = self.Rights.authorized(user, path, "Ww") + parent_permissions = self.Rights.authorized(user, parent_path, "w") + + def prepare(vobject_items, tag=None, write_whole_collection=None): + if (write_whole_collection or + permissions and not parent_permissions): + write_whole_collection = True + content_type = environ.get("CONTENT_TYPE", + "").split(";")[0] + tags = {value: key + for key, value in xmlutils.MIMETYPES.items()} + tag = radicale_item.predict_tag_of_whole_collection( + vobject_items, tags.get(content_type)) + if not tag: + raise ValueError("Can't determine collection tag") + collection_path = pathutils.sanitize_path(path).strip("/") + elif (write_whole_collection is not None and + not write_whole_collection or + not permissions and parent_permissions): + write_whole_collection = False + if tag is None: + tag = storage.predict_tag_of_parent_collection( + vobject_items) + collection_path = posixpath.dirname( + pathutils.sanitize_path(path).strip("/")) + props = None + stored_exc_info = None + items = [] + try: + if tag: + radicale_item.check_and_sanitize_items( + vobject_items, is_collection=write_whole_collection, + tag=tag) + if write_whole_collection and tag == "VCALENDAR": + vobject_components = [] + vobject_item, = vobject_items + for content in ("vevent", "vtodo", "vjournal"): + vobject_components.extend( + getattr(vobject_item, "%s_list" % content, [])) + vobject_components_by_uid = itertools.groupby( + sorted(vobject_components, + key=radicale_item.get_uid), + radicale_item.get_uid) + for uid, components in vobject_components_by_uid: + vobject_collection = vobject.iCalendar() + for component in components: + vobject_collection.add(component) + item = radicale_item.Item( + collection_path=collection_path, + vobject_item=vobject_collection) + item.prepare() + items.append(item) + elif write_whole_collection and tag == "VADDRESSBOOK": + for vobject_item in vobject_items: + item = radicale_item.Item( + collection_path=collection_path, + vobject_item=vobject_item) + item.prepare() + items.append(item) + elif not write_whole_collection: + vobject_item, = vobject_items + item = radicale_item.Item( + collection_path=collection_path, + vobject_item=vobject_item) + item.prepare() + items.append(item) + + if write_whole_collection: + props = {} + if tag: + props["tag"] = tag + if tag == "VCALENDAR" and vobject_items: + if hasattr(vobject_items[0], "x_wr_calname"): + calname = vobject_items[0].x_wr_calname.value + if calname: + props["D:displayname"] = calname + if hasattr(vobject_items[0], "x_wr_caldesc"): + caldesc = vobject_items[0].x_wr_caldesc.value + if caldesc: + props["C:calendar-description"] = caldesc + radicale_item.check_and_sanitize_props(props) + except Exception: + stored_exc_info = sys.exc_info() + + # Use generator for items and delete references to free memory + # early + def items_generator(): + while items: + yield items.pop(0) + + return (items_generator(), tag, write_whole_collection, props, + stored_exc_info) + + try: + vobject_items = tuple(vobject.readComponents(content or "")) + except Exception as e: + logger.warning( + "Bad PUT request on %r: %s", path, e, exc_info=True) + return httputils.BAD_REQUEST + (prepared_items, prepared_tag, prepared_write_whole_collection, + prepared_props, prepared_exc_info) = prepare(vobject_items) + + with self.Collection.acquire_lock("w", user): + item = next(self.Collection.discover(path), None) + parent_item = next(self.Collection.discover(parent_path), None) + if not parent_item: + return httputils.CONFLICT + + write_whole_collection = ( + isinstance(item, storage.BaseCollection) or + not parent_item.get_meta("tag")) + + if write_whole_collection: + tag = prepared_tag + else: + tag = parent_item.get_meta("tag") + + if write_whole_collection: + if not self.Rights.authorized(user, path, "w" if tag else "W"): + return httputils.NOT_ALLOWED + elif not self.Rights.authorized(user, parent_path, "w"): + return httputils.NOT_ALLOWED + + etag = environ.get("HTTP_IF_MATCH", "") + if not item and etag: + # Etag asked but no item found: item has been removed + return httputils.PRECONDITION_FAILED + if item and etag and item.etag != etag: + # Etag asked but item not matching: item has changed + return httputils.PRECONDITION_FAILED + + match = environ.get("HTTP_IF_NONE_MATCH", "") == "*" + if item and match: + # Creation asked but item found: item can't be replaced + return httputils.PRECONDITION_FAILED + + if (tag != prepared_tag or + prepared_write_whole_collection != write_whole_collection): + (prepared_items, prepared_tag, prepared_write_whole_collection, + prepared_props, prepared_exc_info) = prepare( + vobject_items, tag, write_whole_collection) + props = prepared_props + if prepared_exc_info: + logger.warning( + "Bad PUT request on %r: %s", path, prepared_exc_info[1], + exc_info=prepared_exc_info) + return httputils.BAD_REQUEST + + if write_whole_collection: + try: + etag = self.Collection.create_collection( + path, prepared_items, props).etag + except ValueError as e: + logger.warning( + "Bad PUT request on %r: %s", path, e, exc_info=True) + return httputils.BAD_REQUEST + else: + prepared_item, = prepared_items + if (item and item.uid != prepared_item.uid or + not item and parent_item.has_uid(prepared_item.uid)): + return self.webdav_error_response( + "C" if tag == "VCALENDAR" else "CR", + "no-uid-conflict") + + href = posixpath.basename(path.strip("/")) + try: + etag = parent_item.upload(href, prepared_item).etag + except ValueError as e: + logger.warning( + "Bad PUT request on %r: %s", path, e, exc_info=True) + return httputils.BAD_REQUEST + + headers = {"ETag": etag} + return client.CREATED, headers, None diff --git a/radicale/app/report.py b/radicale/app/report.py new file mode 100644 index 00000000..65c7bd79 --- /dev/null +++ b/radicale/app/report.py @@ -0,0 +1,296 @@ +# This file is part of Radicale Server - Calendar Server +# Copyright © 2008 Nicolas Kandel +# Copyright © 2008 Pascal Halter +# Copyright © 2008-2017 Guillaume Ayoub +# Copyright © 2017-2018 Unrud +# +# This library is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Radicale. If not, see . + +""" +Radicale WSGI application. + +Can be used with an external WSGI server or the built-in server. + +""" + +import contextlib +import posixpath +import socket +from http import client +from urllib.parse import unquote, urlparse +from xml.etree import ElementTree as ET + +from radicale import httputils, pathutils, storage, xmlutils +from radicale.item import filter as radicale_filter +from radicale.log import logger + + +def xml_report(base_prefix, path, xml_request, collection, unlock_storage_fn): + """Read and answer REPORT requests. + + Read rfc3253-3.6 for info. + + """ + multistatus = ET.Element(xmlutils.make_tag("D", "multistatus")) + if xml_request is None: + return client.MULTI_STATUS, multistatus + root = xml_request + if root.tag in ( + xmlutils.make_tag("D", "principal-search-property-set"), + xmlutils.make_tag("D", "principal-property-search"), + xmlutils.make_tag("D", "expand-property")): + # We don't support searching for principals or indirect retrieving of + # properties, just return an empty result. + # InfCloud asks for expand-property reports (even if we don't announce + # support for them) and stops working if an error code is returned. + logger.warning("Unsupported REPORT method %r on %r requested", + xmlutils.tag_from_clark(root.tag), path) + return client.MULTI_STATUS, multistatus + if (root.tag == xmlutils.make_tag("C", "calendar-multiget") and + collection.get_meta("tag") != "VCALENDAR" or + root.tag == xmlutils.make_tag("CR", "addressbook-multiget") and + collection.get_meta("tag") != "VADDRESSBOOK" or + root.tag == xmlutils.make_tag("D", "sync-collection") and + collection.get_meta("tag") not in ("VADDRESSBOOK", "VCALENDAR")): + logger.warning("Invalid REPORT method %r on %r requested", + xmlutils.tag_from_clark(root.tag), path) + return (client.CONFLICT, + xmlutils.webdav_error("D", "supported-report")) + prop_element = root.find(xmlutils.make_tag("D", "prop")) + props = ( + [prop.tag for prop in prop_element] + if prop_element is not None else []) + + if root.tag in ( + xmlutils.make_tag("C", "calendar-multiget"), + xmlutils.make_tag("CR", "addressbook-multiget")): + # Read rfc4791-7.9 for info + hreferences = set() + for href_element in root.findall(xmlutils.make_tag("D", "href")): + href_path = pathutils.sanitize_path( + unquote(urlparse(href_element.text).path)) + if (href_path + "/").startswith(base_prefix + "/"): + hreferences.add(href_path[len(base_prefix):]) + else: + logger.warning("Skipping invalid path %r in REPORT request on " + "%r", href_path, path) + elif root.tag == xmlutils.make_tag("D", "sync-collection"): + old_sync_token_element = root.find( + xmlutils.make_tag("D", "sync-token")) + old_sync_token = "" + if old_sync_token_element is not None and old_sync_token_element.text: + old_sync_token = old_sync_token_element.text.strip() + logger.debug("Client provided sync token: %r", old_sync_token) + try: + sync_token, names = collection.sync(old_sync_token) + except ValueError as e: + # Invalid sync token + logger.warning("Client provided invalid sync token %r: %s", + old_sync_token, e, exc_info=True) + return (client.CONFLICT, + xmlutils.webdav_error("D", "valid-sync-token")) + hreferences = ("/" + posixpath.join(collection.path, n) for n in names) + # Append current sync token to response + sync_token_element = ET.Element(xmlutils.make_tag("D", "sync-token")) + sync_token_element.text = sync_token + multistatus.append(sync_token_element) + else: + hreferences = (path,) + filters = ( + root.findall("./%s" % xmlutils.make_tag("C", "filter")) + + root.findall("./%s" % xmlutils.make_tag("CR", "filter"))) + + def retrieve_items(collection, hreferences, multistatus): + """Retrieves all items that are referenced in ``hreferences`` from + ``collection`` and adds 404 responses for missing and invalid items + to ``multistatus``.""" + collection_requested = False + + def get_names(): + """Extracts all names from references in ``hreferences`` and adds + 404 responses for invalid references to ``multistatus``. + If the whole collections is referenced ``collection_requested`` + gets set to ``True``.""" + nonlocal collection_requested + for hreference in hreferences: + try: + name = pathutils.name_from_path(hreference, collection) + except ValueError as e: + logger.warning("Skipping invalid path %r in REPORT request" + " on %r: %s", hreference, path, e) + response = xml_item_response(base_prefix, hreference, + found_item=False) + multistatus.append(response) + continue + if name: + # Reference is an item + yield name + else: + # Reference is a collection + collection_requested = True + + for name, item in collection.get_multi(get_names()): + if not item: + uri = "/" + posixpath.join(collection.path, name) + response = xml_item_response(base_prefix, uri, + found_item=False) + multistatus.append(response) + else: + yield item, False + if collection_requested: + yield from collection.get_all_filtered(filters) + + # Retrieve everything required for finishing the request. + retrieved_items = list(retrieve_items(collection, hreferences, + multistatus)) + collection_tag = collection.get_meta("tag") + # Don't access storage after this! + unlock_storage_fn() + + def match(item, filter_): + tag = collection_tag + if (tag == "VCALENDAR" and + filter_.tag != xmlutils.make_tag("C", filter_)): + if len(filter_) == 0: + return True + if len(filter_) > 1: + raise ValueError("Filter with %d children" % len(filter_)) + if filter_[0].tag != xmlutils.make_tag("C", "comp-filter"): + raise ValueError("Unexpected %r in filter" % filter_[0].tag) + return radicale_filter.comp_match(item, filter_[0]) + if (tag == "VADDRESSBOOK" and + filter_.tag != xmlutils.make_tag("CR", filter_)): + for child in filter_: + if child.tag != xmlutils.make_tag("CR", "prop-filter"): + raise ValueError("Unexpected %r in filter" % child.tag) + test = filter_.get("test", "anyof") + if test == "anyof": + return any( + radicale_filter.prop_match(item.vobject_item, f, "CR") + for f in filter_) + if test == "allof": + return all( + radicale_filter.prop_match(item.vobject_item, f, "CR") + for f in filter_) + raise ValueError("Unsupported filter test: %r" % test) + return all(radicale_filter.prop_match(item.vobject_item, f, "CR") + for f in filter_) + raise ValueError("unsupported filter %r for %r" % (filter_.tag, tag)) + + while retrieved_items: + # ``item.vobject_item`` might be accessed during filtering. + # Don't keep reference to ``item``, because VObject requires a lot of + # memory. + item, filters_matched = retrieved_items.pop(0) + if filters and not filters_matched: + try: + if not all(match(item, filter_) for filter_ in filters): + continue + except ValueError as e: + raise ValueError("Failed to filter item %r from %r: %s" % + (item.href, collection.path, e)) from e + except Exception as e: + raise RuntimeError("Failed to filter item %r from %r: %s" % + (item.href, collection.path, e)) from e + + found_props = [] + not_found_props = [] + + for tag in props: + element = ET.Element(tag) + if tag == xmlutils.make_tag("D", "getetag"): + element.text = item.etag + found_props.append(element) + elif tag == xmlutils.make_tag("D", "getcontenttype"): + element.text = xmlutils.get_content_type(item) + found_props.append(element) + elif tag in ( + xmlutils.make_tag("C", "calendar-data"), + xmlutils.make_tag("CR", "address-data")): + element.text = item.serialize() + found_props.append(element) + else: + not_found_props.append(element) + + uri = "/" + posixpath.join(collection.path, item.href) + multistatus.append(xml_item_response( + base_prefix, uri, found_props=found_props, + not_found_props=not_found_props, found_item=True)) + + return client.MULTI_STATUS, multistatus + + +def xml_item_response(base_prefix, href, found_props=(), not_found_props=(), + found_item=True): + response = ET.Element(xmlutils.make_tag("D", "response")) + + href_tag = ET.Element(xmlutils.make_tag("D", "href")) + href_tag.text = xmlutils.make_href(base_prefix, href) + response.append(href_tag) + + if found_item: + for code, props in ((200, found_props), (404, not_found_props)): + if props: + propstat = ET.Element(xmlutils.make_tag("D", "propstat")) + status = ET.Element(xmlutils.make_tag("D", "status")) + status.text = xmlutils.make_response(code) + prop_tag = ET.Element(xmlutils.make_tag("D", "prop")) + for prop in props: + prop_tag.append(prop) + propstat.append(prop_tag) + propstat.append(status) + response.append(propstat) + else: + status = ET.Element(xmlutils.make_tag("D", "status")) + status.text = xmlutils.make_response(404) + response.append(status) + + return response + + +class ApplicationReportMixin: + def do_REPORT(self, environ, base_prefix, path, user): + """Manage REPORT request.""" + if not self.access(user, path, "r"): + return httputils.NOT_ALLOWED + try: + xml_content = self.read_xml_content(environ) + except RuntimeError as e: + logger.warning( + "Bad REPORT request on %r: %s", path, e, exc_info=True) + return httputils.BAD_REQUEST + except socket.timeout as e: + logger.debug("client timed out", exc_info=True) + return httputils.REQUEST_TIMEOUT + with contextlib.ExitStack() as lock_stack: + lock_stack.enter_context(self.Collection.acquire_lock("r", user)) + item = next(self.Collection.discover(path), None) + if not item: + return httputils.NOT_FOUND + if not self.access(user, path, "r", item): + return httputils.NOT_ALLOWED + if isinstance(item, storage.BaseCollection): + collection = item + else: + collection = item.collection + headers = {"Content-Type": "text/xml; charset=%s" % self.encoding} + try: + status, xml_answer = xml_report( + base_prefix, path, xml_content, collection, + lock_stack.close) + except ValueError as e: + logger.warning( + "Bad REPORT request on %r: %s", path, e, exc_info=True) + return httputils.BAD_REQUEST + return (status, headers, self.write_xml_content(xml_answer)) diff --git a/radicale/auth/__init__.py b/radicale/auth/__init__.py new file mode 100644 index 00000000..02a1802b --- /dev/null +++ b/radicale/auth/__init__.py @@ -0,0 +1,107 @@ +# This file is part of Radicale Server - Calendar Server +# Copyright © 2008 Nicolas Kandel +# Copyright © 2008 Pascal Halter +# Copyright © 2008-2017 Guillaume Ayoub +# Copyright © 2017-2018 Unrud +# +# This library is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Radicale. If not, see . + +""" +Authentication management. + +Default is htpasswd authentication. + +Apache's htpasswd command (httpd.apache.org/docs/programs/htpasswd.html) +manages a file for storing user credentials. It can encrypt passwords using +different methods, e.g. BCRYPT, MD5-APR1 (a version of MD5 modified for +Apache), SHA1, or by using the system's CRYPT routine. The CRYPT and SHA1 +encryption methods implemented by htpasswd are considered as insecure. MD5-APR1 +provides medium security as of 2015. Only BCRYPT can be considered secure by +current standards. + +MD5-APR1-encrypted credentials can be written by all versions of htpasswd (it +is the default, in fact), whereas BCRYPT requires htpasswd 2.4.x or newer. + +The `is_authenticated(user, password)` function provided by this module +verifies the user-given credentials by parsing the htpasswd credential file +pointed to by the ``htpasswd_filename`` configuration value while assuming +the password encryption method specified via the ``htpasswd_encryption`` +configuration value. + +The following htpasswd password encrpytion methods are supported by Radicale +out-of-the-box: + + - plain-text (created by htpasswd -p...) -- INSECURE + - CRYPT (created by htpasswd -d...) -- INSECURE + - SHA1 (created by htpasswd -s...) -- INSECURE + +When passlib (https://pypi.python.org/pypi/passlib) is importable, the +following significantly more secure schemes are parsable by Radicale: + + - MD5-APR1 (htpasswd -m...) -- htpasswd's default method + - BCRYPT (htpasswd -B...) -- Requires htpasswd 2.4.x + +""" + +from importlib import import_module + +from radicale.log import logger + +INTERNAL_TYPES = ("none", "remote_user", "http_x_remote_user", "htpasswd") + + +def load(configuration): + """Load the authentication manager chosen in configuration.""" + auth_type = configuration.get("auth", "type") + if auth_type in INTERNAL_TYPES: + module = "radicale.auth.%s" % auth_type + else: + module = auth_type + try: + class_ = import_module(module).Auth + except Exception as e: + raise RuntimeError("Failed to load authentication module %r: %s" % + (auth_type, e)) from e + logger.info("Authentication type is %r", auth_type) + return class_(configuration) + + +class BaseAuth: + def __init__(self, configuration): + self.configuration = configuration + + def get_external_login(self, environ): + """Optionally provide the login and password externally. + + ``environ`` a dict with the WSGI environment + + If ``()`` is returned, Radicale handles HTTP authentication. + Otherwise, returns a tuple ``(login, password)``. For anonymous users + ``login`` must be ``""``. + + """ + return () + + def login(self, login, password): + """Check credentials and map login to internal user + + ``login`` the login name + + ``password`` the password + + Returns the user name or ``""`` for invalid credentials. + + """ + + raise NotImplementedError diff --git a/radicale/auth.py b/radicale/auth/htpasswd.py similarity index 63% rename from radicale/auth.py rename to radicale/auth/htpasswd.py index ab73a985..c14b4bed 100644 --- a/radicale/auth.py +++ b/radicale/auth/htpasswd.py @@ -2,6 +2,7 @@ # Copyright © 2008 Nicolas Kandel # Copyright © 2008 Pascal Halter # Copyright © 2008-2017 Guillaume Ayoub +# Copyright © 2017-2018 Unrud # # This library is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -16,112 +17,16 @@ # You should have received a copy of the GNU General Public License # along with Radicale. If not, see . -""" -Authentication management. - -Default is htpasswd authentication. - -Apache's htpasswd command (httpd.apache.org/docs/programs/htpasswd.html) -manages a file for storing user credentials. It can encrypt passwords using -different methods, e.g. BCRYPT, MD5-APR1 (a version of MD5 modified for -Apache), SHA1, or by using the system's CRYPT routine. The CRYPT and SHA1 -encryption methods implemented by htpasswd are considered as insecure. MD5-APR1 -provides medium security as of 2015. Only BCRYPT can be considered secure by -current standards. - -MD5-APR1-encrypted credentials can be written by all versions of htpasswd (it -is the default, in fact), whereas BCRYPT requires htpasswd 2.4.x or newer. - -The `is_authenticated(user, password)` function provided by this module -verifies the user-given credentials by parsing the htpasswd credential file -pointed to by the ``htpasswd_filename`` configuration value while assuming -the password encryption method specified via the ``htpasswd_encryption`` -configuration value. - -The following htpasswd password encrpytion methods are supported by Radicale -out-of-the-box: - - - plain-text (created by htpasswd -p...) -- INSECURE - - CRYPT (created by htpasswd -d...) -- INSECURE - - SHA1 (created by htpasswd -s...) -- INSECURE - -When passlib (https://pypi.python.org/pypi/passlib) is importable, the -following significantly more secure schemes are parsable by Radicale: - - - MD5-APR1 (htpasswd -m...) -- htpasswd's default method - - BCRYPT (htpasswd -B...) -- Requires htpasswd 2.4.x - -""" - import base64 import functools import hashlib import hmac import os -from importlib import import_module -from radicale.log import logger - -INTERNAL_TYPES = ("none", "remote_user", "http_x_remote_user", "htpasswd") +from radicale import auth -def load(configuration): - """Load the authentication manager chosen in configuration.""" - auth_type = configuration.get("auth", "type") - if auth_type == "none": - class_ = NoneAuth - elif auth_type == "remote_user": - class_ = RemoteUserAuth - elif auth_type == "http_x_remote_user": - class_ = HttpXRemoteUserAuth - elif auth_type == "htpasswd": - class_ = Auth - else: - try: - class_ = import_module(auth_type).Auth - except Exception as e: - raise RuntimeError("Failed to load authentication module %r: %s" % - (auth_type, e)) from e - logger.info("Authentication type is %r", auth_type) - return class_(configuration) - - -class BaseAuth: - def __init__(self, configuration): - self.configuration = configuration - - def get_external_login(self, environ): - """Optionally provide the login and password externally. - - ``environ`` a dict with the WSGI environment - - If ``()`` is returned, Radicale handles HTTP authentication. - Otherwise, returns a tuple ``(login, password)``. For anonymous users - ``login`` must be ``""``. - - """ - return () - - def login(self, login, password): - """Check credentials and map login to internal user - - ``login`` the login name - - ``password`` the password - - Returns the user name or ``""`` for invalid credentials. - - """ - - raise NotImplementedError - - -class NoneAuth(BaseAuth): - def login(self, login, password): - return login - - -class Auth(BaseAuth): +class Auth(auth.BaseAuth): def __init__(self, configuration): super().__init__(configuration) self.filename = os.path.expanduser( @@ -244,13 +149,3 @@ class Auth(BaseAuth): raise RuntimeError("Failed to load htpasswd file %r: %s" % (self.filename, e)) from e return "" - - -class RemoteUserAuth(NoneAuth): - def get_external_login(self, environ): - return environ.get("REMOTE_USER", ""), "" - - -class HttpXRemoteUserAuth(NoneAuth): - def get_external_login(self, environ): - return environ.get("HTTP_X_REMOTE_USER", ""), "" diff --git a/radicale/auth/http_x_remote_user.py b/radicale/auth/http_x_remote_user.py new file mode 100644 index 00000000..c8f05bbc --- /dev/null +++ b/radicale/auth/http_x_remote_user.py @@ -0,0 +1,25 @@ +# This file is part of Radicale Server - Calendar Server +# Copyright © 2008 Nicolas Kandel +# Copyright © 2008 Pascal Halter +# Copyright © 2008-2017 Guillaume Ayoub +# Copyright © 2017-2018 Unrud +# +# This library is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Radicale. If not, see . + +import radicale.auth.none as none + + +class Auth(none.Auth): + def get_external_login(self, environ): + return environ.get("HTTP_X_REMOTE_USER", ""), "" diff --git a/radicale/auth/none.py b/radicale/auth/none.py new file mode 100644 index 00000000..f1be09e5 --- /dev/null +++ b/radicale/auth/none.py @@ -0,0 +1,25 @@ +# This file is part of Radicale Server - Calendar Server +# Copyright © 2008 Nicolas Kandel +# Copyright © 2008 Pascal Halter +# Copyright © 2008-2017 Guillaume Ayoub +# Copyright © 2017-2018 Unrud +# +# This library is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Radicale. If not, see . + +from radicale import auth + + +class Auth(auth.BaseAuth): + def login(self, login, password): + return login diff --git a/radicale/auth/remote_user.py b/radicale/auth/remote_user.py new file mode 100644 index 00000000..2a9d2bed --- /dev/null +++ b/radicale/auth/remote_user.py @@ -0,0 +1,25 @@ +# This file is part of Radicale Server - Calendar Server +# Copyright © 2008 Nicolas Kandel +# Copyright © 2008 Pascal Halter +# Copyright © 2008-2017 Guillaume Ayoub +# Copyright © 2017-2018 Unrud +# +# This library is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Radicale. If not, see . + +import radicale.auth.none as none + + +class Auth(none.Auth): + def get_external_login(self, environ): + return environ.get("REMOTE_USER", ""), "" diff --git a/radicale/config.py b/radicale/config.py index 97141f78..42966fe9 100644 --- a/radicale/config.py +++ b/radicale/config.py @@ -2,6 +2,7 @@ # Copyright © 2008-2017 Guillaume Ayoub # Copyright © 2008 Nicolas Kandel # Copyright © 2008 Pascal Halter +# Copyright © 2017-2018 Unrud # # This library is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by diff --git a/radicale/httputils.py b/radicale/httputils.py new file mode 100644 index 00000000..1b4438ce --- /dev/null +++ b/radicale/httputils.py @@ -0,0 +1,61 @@ +# This file is part of Radicale Server - Calendar Server +# Copyright © 2008 Nicolas Kandel +# Copyright © 2008 Pascal Halter +# Copyright © 2008-2017 Guillaume Ayoub +# Copyright © 2017-2018 Unrud +# +# This library is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Radicale. If not, see . + +from http import client + +NOT_ALLOWED = ( + client.FORBIDDEN, (("Content-Type", "text/plain"),), + "Access to the requested resource forbidden.") +FORBIDDEN = ( + client.FORBIDDEN, (("Content-Type", "text/plain"),), + "Action on the requested resource refused.") +BAD_REQUEST = ( + client.BAD_REQUEST, (("Content-Type", "text/plain"),), "Bad Request") +NOT_FOUND = ( + client.NOT_FOUND, (("Content-Type", "text/plain"),), + "The requested resource could not be found.") +CONFLICT = ( + client.CONFLICT, (("Content-Type", "text/plain"),), + "Conflict in the request.") +WEBDAV_PRECONDITION_FAILED = ( + client.CONFLICT, (("Content-Type", "text/plain"),), + "WebDAV precondition failed.") +METHOD_NOT_ALLOWED = ( + client.METHOD_NOT_ALLOWED, (("Content-Type", "text/plain"),), + "The method is not allowed on the requested resource.") +PRECONDITION_FAILED = ( + client.PRECONDITION_FAILED, + (("Content-Type", "text/plain"),), "Precondition failed.") +REQUEST_TIMEOUT = ( + client.REQUEST_TIMEOUT, (("Content-Type", "text/plain"),), + "Connection timed out.") +REQUEST_ENTITY_TOO_LARGE = ( + client.REQUEST_ENTITY_TOO_LARGE, (("Content-Type", "text/plain"),), + "Request body too large.") +REMOTE_DESTINATION = ( + client.BAD_GATEWAY, (("Content-Type", "text/plain"),), + "Remote destination not supported.") +DIRECTORY_LISTING = ( + client.FORBIDDEN, (("Content-Type", "text/plain"),), + "Directory listings are not supported.") +INTERNAL_SERVER_ERROR = ( + client.INTERNAL_SERVER_ERROR, (("Content-Type", "text/plain"),), + "A server error occurred. Please contact the administrator.") + +DAV_HEADERS = "1, 2, 3, calendar-access, addressbook, extended-mkcol" diff --git a/radicale/item/__init__.py b/radicale/item/__init__.py new file mode 100644 index 00000000..d1bcc6d8 --- /dev/null +++ b/radicale/item/__init__.py @@ -0,0 +1,374 @@ +# This file is part of Radicale Server - Calendar Server +# Copyright © 2008 Nicolas Kandel +# Copyright © 2008 Pascal Halter +# Copyright © 2014 Jean-Marc Martins +# Copyright © 2008-2017 Guillaume Ayoub +# Copyright © 2017-2018 Unrud +# +# This library is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Radicale. If not, see . + +import math +import sys +from hashlib import md5 +from random import getrandbits + +import vobject + +from radicale.item import filter as radicale_filter + + +def predict_tag_of_parent_collection(vobject_items): + if len(vobject_items) != 1: + return "" + if vobject_items[0].name == "VCALENDAR": + return "VCALENDAR" + if vobject_items[0].name in ("VCARD", "VLIST"): + return "VADDRESSBOOK" + return "" + + +def predict_tag_of_whole_collection(vobject_items, fallback_tag=None): + if vobject_items and vobject_items[0].name == "VCALENDAR": + return "VCALENDAR" + if vobject_items and vobject_items[0].name in ("VCARD", "VLIST"): + return "VADDRESSBOOK" + if not fallback_tag and not vobject_items: + # Maybe an empty address book + return "VADDRESSBOOK" + return fallback_tag + + +def check_and_sanitize_items(vobject_items, is_collection=False, tag=None): + """Check vobject items for common errors and add missing UIDs. + + ``is_collection`` indicates that vobject_item contains unrelated + components. + + The ``tag`` of the collection. + + """ + if tag and tag not in ("VCALENDAR", "VADDRESSBOOK"): + raise ValueError("Unsupported collection tag: %r" % tag) + if not is_collection and len(vobject_items) != 1: + raise ValueError("Item contains %d components" % len(vobject_items)) + if tag == "VCALENDAR": + if len(vobject_items) > 1: + raise RuntimeError("VCALENDAR collection contains %d " + "components" % len(vobject_items)) + vobject_item = vobject_items[0] + if vobject_item.name != "VCALENDAR": + raise ValueError("Item type %r not supported in %r " + "collection" % (vobject_item.name, tag)) + component_uids = set() + for component in vobject_item.components(): + if component.name in ("VTODO", "VEVENT", "VJOURNAL"): + component_uid = get_uid(component) + if component_uid: + component_uids.add(component_uid) + component_name = None + object_uid = None + object_uid_set = False + for component in vobject_item.components(): + # https://tools.ietf.org/html/rfc4791#section-4.1 + if component.name == "VTIMEZONE": + continue + if component_name is None or is_collection: + component_name = component.name + elif component_name != component.name: + raise ValueError("Multiple component types in object: %r, %r" % + (component_name, component.name)) + if component_name not in ("VTODO", "VEVENT", "VJOURNAL"): + continue + component_uid = get_uid(component) + if not object_uid_set or is_collection: + object_uid_set = True + object_uid = component_uid + if not component_uid: + if not is_collection: + raise ValueError("%s component without UID in object" % + component_name) + component_uid = find_available_uid( + component_uids.__contains__) + component_uids.add(component_uid) + if hasattr(component, "uid"): + component.uid.value = component_uid + else: + component.add("UID").value = component_uid + elif not object_uid or not component_uid: + raise ValueError("Multiple %s components without UID in " + "object" % component_name) + elif object_uid != component_uid: + raise ValueError( + "Multiple %s components with different UIDs in object: " + "%r, %r" % (component_name, object_uid, component_uid)) + # vobject interprets recurrence rules on demand + try: + component.rruleset + except Exception as e: + raise ValueError("invalid recurrence rules in %s" % + component.name) from e + elif tag == "VADDRESSBOOK": + # https://tools.ietf.org/html/rfc6352#section-5.1 + object_uids = set() + for vobject_item in vobject_items: + if vobject_item.name == "VCARD": + object_uid = get_uid(vobject_item) + if object_uid: + object_uids.add(object_uid) + for vobject_item in vobject_items: + if vobject_item.name == "VLIST": + # Custom format used by SOGo Connector to store lists of + # contacts + continue + if vobject_item.name != "VCARD": + raise ValueError("Item type %r not supported in %r " + "collection" % (vobject_item.name, tag)) + object_uid = get_uid(vobject_item) + if not object_uid: + if not is_collection: + raise ValueError("%s object without UID" % + vobject_item.name) + object_uid = find_available_uid(object_uids.__contains__) + object_uids.add(object_uid) + if hasattr(vobject_item, "uid"): + vobject_item.uid.value = object_uid + else: + vobject_item.add("UID").value = object_uid + else: + for i in vobject_items: + raise ValueError("Item type %r not supported in %s collection" % + (i.name, repr(tag) if tag else "generic")) + + +def check_and_sanitize_props(props): + """Check collection properties for common errors.""" + tag = props.get("tag") + if tag and tag not in ("VCALENDAR", "VADDRESSBOOK"): + raise ValueError("Unsupported collection tag: %r" % tag) + + +def find_available_uid(exists_fn, suffix=""): + """Generate a pseudo-random UID""" + # Prevent infinite loop + for _ in range(1000): + r = "%016x" % getrandbits(128) + name = "%s-%s-%s-%s-%s%s" % ( + r[:8], r[8:12], r[12:16], r[16:20], r[20:], suffix) + if not exists_fn(name): + return name + # something is wrong with the PRNG + raise RuntimeError("No unique random sequence found") + + +def get_etag(text): + """Etag from collection or item. + + Encoded as quoted-string (see RFC 2616). + + """ + etag = md5() + etag.update(text.encode("utf-8")) + return '"%s"' % etag.hexdigest() + + +def get_uid(vobject_component): + """UID value of an item if defined.""" + return (vobject_component.uid.value + if hasattr(vobject_component, "uid") else None) + + +def get_uid_from_object(vobject_item): + """UID value of an calendar/addressbook object.""" + if vobject_item.name == "VCALENDAR": + if hasattr(vobject_item, "vevent"): + return get_uid(vobject_item.vevent) + if hasattr(vobject_item, "vjournal"): + return get_uid(vobject_item.vjournal) + if hasattr(vobject_item, "vtodo"): + return get_uid(vobject_item.vtodo) + elif vobject_item.name == "VCARD": + return get_uid(vobject_item) + return None + + +def find_tag(vobject_item): + """Find component name from ``vobject_item``.""" + if vobject_item.name == "VCALENDAR": + for component in vobject_item.components(): + if component.name != "VTIMEZONE": + return component.name or "" + return "" + + +def find_tag_and_time_range(vobject_item): + """Find component name and enclosing time range from ``vobject item``. + + Returns a tuple (``tag``, ``start``, ``end``) where ``tag`` is a string + and ``start`` and ``end`` are POSIX timestamps (as int). + + This is intened to be used for matching against simplified prefilters. + + """ + tag = find_tag(vobject_item) + if not tag: + return ( + tag, radicale_filter.TIMESTAMP_MIN, radicale_filter.TIMESTAMP_MAX) + start = end = None + + def range_fn(range_start, range_end, is_recurrence): + nonlocal start, end + if start is None or range_start < start: + start = range_start + if end is None or end < range_end: + end = range_end + return False + + def infinity_fn(range_start): + nonlocal start, end + if start is None or range_start < start: + start = range_start + end = radicale_filter.DATETIME_MAX + return True + + radicale_filter.visit_time_ranges(vobject_item, tag, range_fn, infinity_fn) + if start is None: + start = radicale_filter.DATETIME_MIN + if end is None: + end = radicale_filter.DATETIME_MAX + try: + return tag, math.floor(start.timestamp()), math.ceil(end.timestamp()) + except ValueError as e: + if str(e) == ("offset must be a timedelta representing a whole " + "number of minutes") and sys.version_info < (3, 6): + raise RuntimeError("Unsupported in Python < 3.6: %s" % e) from e + raise + + +class Item: + def __init__(self, collection_path=None, collection=None, + vobject_item=None, href=None, last_modified=None, text=None, + etag=None, uid=None, name=None, component_name=None, + time_range=None): + """Initialize an item. + + ``collection_path`` the path of the parent collection (optional if + ``collection`` is set). + + ``collection`` the parent collection (optional). + + ``href`` the href of the item. + + ``last_modified`` the HTTP-datetime of when the item was modified. + + ``text`` the text representation of the item (optional if + ``vobject_item`` is set). + + ``vobject_item`` the vobject item (optional if ``text`` is set). + + ``etag`` the etag of the item (optional). See ``get_etag``. + + ``uid`` the UID of the object (optional). See ``get_uid_from_object``. + + ``name`` the name of the item (optional). See ``vobject_item.name``. + + ``component_name`` the name of the primary component (optional). + See ``find_tag``. + + ``time_range`` the enclosing time range. + See ``find_tag_and_time_range``. + + """ + if text is None and vobject_item is None: + raise ValueError( + "at least one of 'text' or 'vobject_item' must be set") + if collection_path is None: + if collection is None: + raise ValueError("at least one of 'collection_path' or " + "'collection' must be set") + collection_path = collection.path + self._collection_path = collection_path + self.collection = collection + self.href = href + self.last_modified = last_modified + self._text = text + self._vobject_item = vobject_item + self._etag = etag + self._uid = uid + self._name = name + self._component_name = component_name + self._time_range = time_range + + def serialize(self): + if self._text is None: + try: + self._text = self.vobject_item.serialize() + except Exception as e: + raise RuntimeError("Failed to serialize item %r from %r: %s" % + (self.href, self._collection_path, + e)) from e + return self._text + + @property + def vobject_item(self): + if self._vobject_item is None: + try: + self._vobject_item = vobject.readOne(self._text) + except Exception as e: + raise RuntimeError("Failed to parse item %r from %r: %s" % + (self.href, self._collection_path, + e)) from e + return self._vobject_item + + @property + def etag(self): + """Encoded as quoted-string (see RFC 2616).""" + if self._etag is None: + self._etag = get_etag(self.serialize()) + return self._etag + + @property + def uid(self): + if self._uid is None: + self._uid = get_uid_from_object(self.vobject_item) + return self._uid + + @property + def name(self): + if self._name is None: + self._name = self.vobject_item.name or "" + return self._name + + @property + def component_name(self): + if self._component_name is not None: + return self._component_name + return find_tag(self.vobject_item) + + @property + def time_range(self): + if self._time_range is None: + self._component_name, *self._time_range = ( + find_tag_and_time_range(self.vobject_item)) + return self._time_range + + def prepare(self): + """Fill cache with values.""" + orig_vobject_item = self._vobject_item + self.serialize() + self.etag + self.uid + self.name + self.time_range + self.component_name + self._vobject_item = orig_vobject_item diff --git a/radicale/item/filter.py b/radicale/item/filter.py new file mode 100644 index 00000000..4d38d706 --- /dev/null +++ b/radicale/item/filter.py @@ -0,0 +1,529 @@ +# This file is part of Radicale Server - Calendar Server +# Copyright © 2008 Nicolas Kandel +# Copyright © 2008 Pascal Halter +# Copyright © 2008-2015 Guillaume Ayoub +# Copyright © 2017-2018 Unrud +# +# This library is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Radicale. If not, see . + + +import math +from datetime import date, datetime, timedelta, timezone +from itertools import chain + +from radicale import xmlutils +from radicale.log import logger + +DAY = timedelta(days=1) +SECOND = timedelta(seconds=1) +DATETIME_MIN = datetime.min.replace(tzinfo=timezone.utc) +DATETIME_MAX = datetime.max.replace(tzinfo=timezone.utc) +TIMESTAMP_MIN = math.floor(DATETIME_MIN.timestamp()) +TIMESTAMP_MAX = math.ceil(DATETIME_MAX.timestamp()) + + +def date_to_datetime(date_): + """Transform a date to a UTC datetime. + + If date_ is a datetime without timezone, return as UTC datetime. If date_ + is already a datetime with timezone, return as is. + + """ + if not isinstance(date_, datetime): + date_ = datetime.combine(date_, datetime.min.time()) + if not date_.tzinfo: + date_ = date_.replace(tzinfo=timezone.utc) + return date_ + + +def comp_match(item, filter_, level=0): + """Check whether the ``item`` matches the comp ``filter_``. + + If ``level`` is ``0``, the filter is applied on the + item's collection. Otherwise, it's applied on the item. + + See rfc4791-9.7.1. + + """ + + # TODO: Filtering VALARM and VFREEBUSY is not implemented + # HACK: the filters are tested separately against all components + + if level == 0: + tag = item.name + elif level == 1: + tag = item.component_name + else: + logger.warning( + "Filters with three levels of comp-filter are not supported") + return True + if not tag: + return False + name = filter_.get("name").upper() + if len(filter_) == 0: + # Point #1 of rfc4791-9.7.1 + return name == tag + if len(filter_) == 1: + if filter_[0].tag == xmlutils.make_tag("C", "is-not-defined"): + # Point #2 of rfc4791-9.7.1 + return name != tag + if name != tag: + return False + if (level == 0 and name != "VCALENDAR" or + level == 1 and name not in ("VTODO", "VEVENT", "VJOURNAL")): + logger.warning("Filtering %s is not supported" % name) + return True + # Point #3 and #4 of rfc4791-9.7.1 + components = ([item.vobject_item] if level == 0 + else list(getattr(item.vobject_item, + "%s_list" % tag.lower()))) + for child in filter_: + if child.tag == xmlutils.make_tag("C", "prop-filter"): + if not any(prop_match(comp, child, "C") + for comp in components): + return False + elif child.tag == xmlutils.make_tag("C", "time-range"): + if not time_range_match(item.vobject_item, filter_[0], tag): + return False + elif child.tag == xmlutils.make_tag("C", "comp-filter"): + if not comp_match(item, child, level=level + 1): + return False + else: + raise ValueError("Unexpected %r in comp-filter" % child.tag) + return True + + +def prop_match(vobject_item, filter_, ns): + """Check whether the ``item`` matches the prop ``filter_``. + + See rfc4791-9.7.2 and rfc6352-10.5.1. + + """ + name = filter_.get("name").lower() + if len(filter_) == 0: + # Point #1 of rfc4791-9.7.2 + return name in vobject_item.contents + if len(filter_) == 1: + if filter_[0].tag == xmlutils.make_tag("C", "is-not-defined"): + # Point #2 of rfc4791-9.7.2 + return name not in vobject_item.contents + if name not in vobject_item.contents: + return False + # Point #3 and #4 of rfc4791-9.7.2 + for child in filter_: + if ns == "C" and child.tag == xmlutils.make_tag("C", "time-range"): + if not time_range_match(vobject_item, child, name): + return False + elif child.tag == xmlutils.make_tag(ns, "text-match"): + if not text_match(vobject_item, child, name, ns): + return False + elif child.tag == xmlutils.make_tag(ns, "param-filter"): + if not param_filter_match(vobject_item, child, name, ns): + return False + else: + raise ValueError("Unexpected %r in prop-filter" % child.tag) + return True + + +def time_range_match(vobject_item, filter_, child_name): + """Check whether the component/property ``child_name`` of + ``vobject_item`` matches the time-range ``filter_``.""" + + start = filter_.get("start") + end = filter_.get("end") + if not start and not end: + return False + if start: + start = datetime.strptime(start, "%Y%m%dT%H%M%SZ") + else: + start = datetime.min + if end: + end = datetime.strptime(end, "%Y%m%dT%H%M%SZ") + else: + end = datetime.max + start = start.replace(tzinfo=timezone.utc) + end = end.replace(tzinfo=timezone.utc) + + matched = False + + def range_fn(range_start, range_end, is_recurrence): + nonlocal matched + if start < range_end and range_start < end: + matched = True + return True + if end < range_start and not is_recurrence: + return True + return False + + def infinity_fn(start): + return False + + visit_time_ranges(vobject_item, child_name, range_fn, infinity_fn) + return matched + + +def visit_time_ranges(vobject_item, child_name, range_fn, infinity_fn): + """Visit all time ranges in the component/property ``child_name`` of + `vobject_item`` with visitors ``range_fn`` and ``infinity_fn``. + + ``range_fn`` gets called for every time_range with ``start`` and ``end`` + datetimes and ``is_recurrence`` as arguments. If the function returns True, + the operation is cancelled. + + ``infinity_fn`` gets called when an infiite recurrence rule is detected + with ``start`` datetime as argument. If the function returns True, the + operation is cancelled. + + See rfc4791-9.9. + + """ + + # HACK: According to rfc5545-3.8.4.4 an recurrance that is resheduled + # with Recurrence ID affects the recurrence itself and all following + # recurrences too. This is not respected and client don't seem to bother + # either. + + def getrruleset(child, ignore=()): + if (hasattr(child, "rrule") and + ";UNTIL=" not in child.rrule.value.upper() and + ";COUNT=" not in child.rrule.value.upper()): + for dtstart in child.getrruleset(addRDate=True): + if dtstart in ignore: + continue + if infinity_fn(date_to_datetime(dtstart)): + return (), True + break + return filter(lambda dtstart: dtstart not in ignore, + child.getrruleset(addRDate=True)), False + + def get_children(components): + main = None + recurrences = [] + for comp in components: + if hasattr(comp, "recurrence_id") and comp.recurrence_id.value: + recurrences.append(comp.recurrence_id.value) + if comp.rruleset: + # Prevent possible infinite loop + raise ValueError("Overwritten recurrence with RRULESET") + yield comp, True, () + else: + if main is not None: + raise ValueError("Multiple main components") + main = comp + if main is None: + raise ValueError("Main component missing") + yield main, False, recurrences + + # Comments give the lines in the tables of the specification + if child_name == "VEVENT": + for child, is_recurrence, recurrences in get_children( + vobject_item.vevent_list): + # TODO: check if there's a timezone + dtstart = child.dtstart.value + + if child.rruleset: + dtstarts, infinity = getrruleset(child, recurrences) + if infinity: + return + else: + dtstarts = (dtstart,) + + dtend = getattr(child, "dtend", None) + if dtend is not None: + dtend = dtend.value + original_duration = (dtend - dtstart).total_seconds() + dtend = date_to_datetime(dtend) + + duration = getattr(child, "duration", None) + if duration is not None: + original_duration = duration = duration.value + + for dtstart in dtstarts: + dtstart_is_datetime = isinstance(dtstart, datetime) + dtstart = date_to_datetime(dtstart) + + if dtend is not None: + # Line 1 + dtend = dtstart + timedelta(seconds=original_duration) + if range_fn(dtstart, dtend, is_recurrence): + return + elif duration is not None: + if original_duration is None: + original_duration = duration.seconds + if duration.seconds > 0: + # Line 2 + if range_fn(dtstart, dtstart + duration, + is_recurrence): + return + else: + # Line 3 + if range_fn(dtstart, dtstart + SECOND, is_recurrence): + return + elif dtstart_is_datetime: + # Line 4 + if range_fn(dtstart, dtstart + SECOND, is_recurrence): + return + else: + # Line 5 + if range_fn(dtstart, dtstart + DAY, is_recurrence): + return + + elif child_name == "VTODO": + for child, is_recurrence, recurrences in get_children( + vobject_item.vtodo_list): + dtstart = getattr(child, "dtstart", None) + duration = getattr(child, "duration", None) + due = getattr(child, "due", None) + completed = getattr(child, "completed", None) + created = getattr(child, "created", None) + + if dtstart is not None: + dtstart = date_to_datetime(dtstart.value) + if duration is not None: + duration = duration.value + if due is not None: + due = date_to_datetime(due.value) + if dtstart is not None: + original_duration = (due - dtstart).total_seconds() + if completed is not None: + completed = date_to_datetime(completed.value) + if created is not None: + created = date_to_datetime(created.value) + original_duration = (completed - created).total_seconds() + elif created is not None: + created = date_to_datetime(created.value) + + if child.rruleset: + reference_dates, infinity = getrruleset(child, recurrences) + if infinity: + return + else: + if dtstart is not None: + reference_dates = (dtstart,) + elif due is not None: + reference_dates = (due,) + elif completed is not None: + reference_dates = (completed,) + elif created is not None: + reference_dates = (created,) + else: + # Line 8 + if range_fn(DATETIME_MIN, DATETIME_MAX, is_recurrence): + return + reference_dates = () + + for reference_date in reference_dates: + reference_date = date_to_datetime(reference_date) + + if dtstart is not None and duration is not None: + # Line 1 + if range_fn(reference_date, + reference_date + duration + SECOND, + is_recurrence): + return + if range_fn(reference_date + duration - SECOND, + reference_date + duration + SECOND, + is_recurrence): + return + elif dtstart is not None and due is not None: + # Line 2 + due = reference_date + timedelta(seconds=original_duration) + if (range_fn(reference_date, due, is_recurrence) or + range_fn(reference_date, + reference_date + SECOND, is_recurrence) or + range_fn(due - SECOND, due, is_recurrence) or + range_fn(due - SECOND, reference_date + SECOND, + is_recurrence)): + return + elif dtstart is not None: + if range_fn(reference_date, reference_date + SECOND, + is_recurrence): + return + elif due is not None: + # Line 4 + if range_fn(reference_date - SECOND, reference_date, + is_recurrence): + return + elif completed is not None and created is not None: + # Line 5 + completed = reference_date + timedelta( + seconds=original_duration) + if (range_fn(reference_date - SECOND, + reference_date + SECOND, + is_recurrence) or + range_fn(completed - SECOND, completed + SECOND, + is_recurrence) or + range_fn(reference_date - SECOND, + reference_date + SECOND, is_recurrence) or + range_fn(completed - SECOND, completed + SECOND, + is_recurrence)): + return + elif completed is not None: + # Line 6 + if range_fn(reference_date - SECOND, + reference_date + SECOND, is_recurrence): + return + elif created is not None: + # Line 7 + if range_fn(reference_date, DATETIME_MAX, is_recurrence): + return + + elif child_name == "VJOURNAL": + for child, is_recurrence, recurrences in get_children( + vobject_item.vjournal_list): + dtstart = getattr(child, "dtstart", None) + + if dtstart is not None: + dtstart = dtstart.value + if child.rruleset: + dtstarts, infinity = getrruleset(child, recurrences) + if infinity: + return + else: + dtstarts = (dtstart,) + + for dtstart in dtstarts: + dtstart_is_datetime = isinstance(dtstart, datetime) + dtstart = date_to_datetime(dtstart) + + if dtstart_is_datetime: + # Line 1 + if range_fn(dtstart, dtstart + SECOND, is_recurrence): + return + else: + # Line 2 + if range_fn(dtstart, dtstart + DAY, is_recurrence): + return + + else: + # Match a property + child = getattr(vobject_item, child_name.lower()) + if isinstance(child, date): + range_fn(child, child + DAY, False) + elif isinstance(child, datetime): + range_fn(child, child + SECOND, False) + + +def text_match(vobject_item, filter_, child_name, ns, attrib_name=None): + """Check whether the ``item`` matches the text-match ``filter_``. + + See rfc4791-9.7.5. + + """ + # TODO: collations are not supported, but the default ones needed + # for DAV servers are actually pretty useless. Texts are lowered to + # be case-insensitive, almost as the "i;ascii-casemap" value. + text = next(filter_.itertext()).lower() + match_type = "contains" + if ns == "CR": + match_type = filter_.get("match-type", match_type) + + def match(value): + value = value.lower() + if match_type == "equals": + return value == text + if match_type == "contains": + return text in value + if match_type == "starts-with": + return value.startswith(text) + if match_type == "ends-with": + return value.endswith(text) + raise ValueError("Unexpected text-match match-type: %r" % match_type) + + children = getattr(vobject_item, "%s_list" % child_name, []) + if attrib_name: + condition = any( + match(attrib) for child in children + for attrib in child.params.get(attrib_name, [])) + else: + condition = any(match(child.value) for child in children) + if filter_.get("negate-condition") == "yes": + return not condition + else: + return condition + + +def param_filter_match(vobject_item, filter_, parent_name, ns): + """Check whether the ``item`` matches the param-filter ``filter_``. + + See rfc4791-9.7.3. + + """ + name = filter_.get("name").upper() + children = getattr(vobject_item, "%s_list" % parent_name, []) + condition = any(name in child.params for child in children) + if len(filter_): + if filter_[0].tag == xmlutils.make_tag(ns, "text-match"): + return condition and text_match( + vobject_item, filter_[0], parent_name, ns, name) + elif filter_[0].tag == xmlutils.make_tag(ns, "is-not-defined"): + return not condition + else: + return condition + + +def simplify_prefilters(filters, collection_tag="VCALENDAR"): + """Creates a simplified condition from ``filters``. + + Returns a tuple (``tag``, ``start``, ``end``, ``simple``) where ``tag`` is + a string or None (match all) and ``start`` and ``end`` are POSIX + timestamps (as int). ``simple`` is a bool that indicates that ``filters`` + and the simplified condition are identical. + + """ + flat_filters = tuple(chain.from_iterable(filters)) + simple = len(flat_filters) <= 1 + for col_filter in flat_filters: + if collection_tag != "VCALENDAR": + simple = False + break + if (col_filter.tag != xmlutils.make_tag("C", "comp-filter") or + col_filter.get("name").upper() != "VCALENDAR"): + simple = False + continue + simple &= len(col_filter) <= 1 + for comp_filter in col_filter: + if comp_filter.tag != xmlutils.make_tag("C", "comp-filter"): + simple = False + continue + tag = comp_filter.get("name").upper() + if comp_filter.find( + xmlutils.make_tag("C", "is-not-defined")) is not None: + simple = False + continue + simple &= len(comp_filter) <= 1 + for time_filter in comp_filter: + if tag not in ("VTODO", "VEVENT", "VJOURNAL"): + simple = False + break + if time_filter.tag != xmlutils.make_tag("C", "time-range"): + simple = False + continue + start = time_filter.get("start") + end = time_filter.get("end") + if start: + start = math.floor(datetime.strptime( + start, "%Y%m%dT%H%M%SZ").replace( + tzinfo=timezone.utc).timestamp()) + else: + start = TIMESTAMP_MIN + if end: + end = math.ceil(datetime.strptime( + end, "%Y%m%dT%H%M%SZ").replace( + tzinfo=timezone.utc).timestamp()) + else: + end = TIMESTAMP_MAX + return tag, start, end, simple + return tag, TIMESTAMP_MIN, TIMESTAMP_MAX, simple + return None, TIMESTAMP_MIN, TIMESTAMP_MAX, simple diff --git a/radicale/log.py b/radicale/log.py index f17aab20..598f5f2d 100644 --- a/radicale/log.py +++ b/radicale/log.py @@ -1,5 +1,6 @@ # This file is part of Radicale Server - Calendar Server # Copyright © 2011-2017 Guillaume Ayoub +# Copyright © 2017-2018 Unrud # # This library is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by diff --git a/radicale/pathutils.py b/radicale/pathutils.py new file mode 100644 index 00000000..9afdf67f --- /dev/null +++ b/radicale/pathutils.py @@ -0,0 +1,217 @@ +# This file is part of Radicale Server - Calendar Server +# Copyright © 2014 Jean-Marc Martins +# Copyright © 2012-2017 Guillaume Ayoub +# Copyright © 2017-2018 Unrud +# +# This library is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Radicale. If not, see . + +import os +import posixpath +import threading +from contextlib import contextmanager + +if os.name == "nt": + import ctypes + import ctypes.wintypes + import msvcrt + + LOCKFILE_EXCLUSIVE_LOCK = 2 + if ctypes.sizeof(ctypes.c_void_p) == 4: + ULONG_PTR = ctypes.c_uint32 + else: + ULONG_PTR = ctypes.c_uint64 + + class Overlapped(ctypes.Structure): + _fields_ = [ + ("internal", ULONG_PTR), + ("internal_high", ULONG_PTR), + ("offset", ctypes.wintypes.DWORD), + ("offset_high", ctypes.wintypes.DWORD), + ("h_event", ctypes.wintypes.HANDLE)] + + lock_file_ex = ctypes.windll.kernel32.LockFileEx + lock_file_ex.argtypes = [ + ctypes.wintypes.HANDLE, + ctypes.wintypes.DWORD, + ctypes.wintypes.DWORD, + ctypes.wintypes.DWORD, + ctypes.wintypes.DWORD, + ctypes.POINTER(Overlapped)] + lock_file_ex.restype = ctypes.wintypes.BOOL + unlock_file_ex = ctypes.windll.kernel32.UnlockFileEx + unlock_file_ex.argtypes = [ + ctypes.wintypes.HANDLE, + ctypes.wintypes.DWORD, + ctypes.wintypes.DWORD, + ctypes.wintypes.DWORD, + ctypes.POINTER(Overlapped)] + unlock_file_ex.restype = ctypes.wintypes.BOOL +elif os.name == "posix": + import fcntl + + +class RwLock: + """A readers-Writer lock that locks a file.""" + + def __init__(self, path): + self._path = path + self._readers = 0 + self._writer = False + self._lock = threading.Lock() + + @property + def locked(self): + with self._lock: + if self._readers > 0: + return "r" + if self._writer: + return "w" + return "" + + @contextmanager + def acquire(self, mode): + if mode not in "rw": + raise ValueError("Invalid mode: %r" % mode) + with open(self._path, "w+") as lock_file: + if os.name == "nt": + handle = msvcrt.get_osfhandle(lock_file.fileno()) + flags = LOCKFILE_EXCLUSIVE_LOCK if mode == "w" else 0 + overlapped = Overlapped() + if not lock_file_ex(handle, flags, 0, 1, 0, overlapped): + raise RuntimeError("Locking the storage failed: %s" % + ctypes.FormatError()) + elif os.name == "posix": + _cmd = fcntl.LOCK_EX if mode == "w" else fcntl.LOCK_SH + try: + fcntl.flock(lock_file.fileno(), _cmd) + except OSError as e: + raise RuntimeError("Locking the storage failed: %s" % + e) from e + else: + raise RuntimeError("Locking the storage failed: " + "Unsupported operating system") + with self._lock: + if self._writer or mode == "w" and self._readers != 0: + raise RuntimeError("Locking the storage failed: " + "Guarantees failed") + if mode == "r": + self._readers += 1 + else: + self._writer = True + try: + yield + finally: + with self._lock: + if mode == "r": + self._readers -= 1 + self._writer = False + + +def fsync(fd): + if os.name == "posix" and hasattr(fcntl, "F_FULLFSYNC"): + fcntl.fcntl(fd, fcntl.F_FULLFSYNC) + else: + os.fsync(fd) + + +def sanitize_path(path): + """Make path absolute with leading slash to prevent access to other data. + + Preserve potential trailing slash. + + """ + trailing_slash = "/" if path.endswith("/") else "" + path = posixpath.normpath(path) + new_path = "/" + for part in path.split("/"): + if not is_safe_path_component(part): + continue + new_path = posixpath.join(new_path, part) + trailing_slash = "" if new_path.endswith("/") else trailing_slash + return new_path + trailing_slash + + +def is_safe_path_component(path): + """Check if path is a single component of a path. + + Check that the path is safe to join too. + + """ + return path and "/" not in path and path not in (".", "..") + + +def is_safe_filesystem_path_component(path): + """Check if path is a single component of a local and posix filesystem + path. + + Check that the path is safe to join too. + + """ + return ( + path and not os.path.splitdrive(path)[0] and + not os.path.split(path)[0] and path not in (os.curdir, os.pardir) and + not path.startswith(".") and not path.endswith("~") and + is_safe_path_component(path)) + + +def path_to_filesystem(root, *paths): + """Convert path to a local filesystem path relative to base_folder. + + `root` must be a secure filesystem path, it will be prepend to the path. + + Conversion of `paths` is done in a secure manner, or raises ``ValueError``. + + """ + paths = [sanitize_path(path).strip("/") for path in paths] + safe_path = root + for path in paths: + if not path: + continue + for part in path.split("/"): + if not is_safe_filesystem_path_component(part): + raise UnsafePathError(part) + safe_path_parent = safe_path + safe_path = os.path.join(safe_path, part) + # Check for conflicting files (e.g. case-insensitive file systems + # or short names on Windows file systems) + if (os.path.lexists(safe_path) and + part not in (e.name for e in + os.scandir(safe_path_parent))): + raise CollidingPathError(part) + return safe_path + + +class UnsafePathError(ValueError): + def __init__(self, path): + message = "Can't translate name safely to filesystem: %r" % path + super().__init__(message) + + +class CollidingPathError(ValueError): + def __init__(self, path): + message = "File name collision: %r" % path + super().__init__(message) + + +def name_from_path(path, collection): + """Return Radicale item name from ``path``.""" + path = path.strip("/") + "/" + start = collection.path + "/" + if not path.startswith(start): + raise ValueError("%r doesn't start with %r" % (path, start)) + name = path[len(start):][:-1] + if name and not is_safe_path_component(name): + raise ValueError("%r is not a component in collection %r" % + (name, collection.path)) + return name diff --git a/radicale/rights.py b/radicale/rights.py deleted file mode 100644 index eb7a46b4..00000000 --- a/radicale/rights.py +++ /dev/null @@ -1,188 +0,0 @@ -# This file is part of Radicale Server - Calendar Server -# Copyright © 2012-2017 Guillaume Ayoub -# -# This library is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This library is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with Radicale. If not, see . - -""" -Rights backends. - -This module loads the rights backend, according to the rights -configuration. - -Default rights are based on a regex-based file whose name is specified in the -config (section "right", key "file"). - -Authentication login is matched against the "user" key, and collection's path -is matched against the "collection" key. You can use Python's ConfigParser -interpolation values %(login)s and %(path)s. You can also get groups from the -user regex in the collection with {0}, {1}, etc. - -For example, for the "user" key, ".+" means "authenticated user" and ".*" -means "anybody" (including anonymous users). - -Section names are only used for naming the rule. - -Leading or ending slashes are trimmed from collection's path. - -""" - -import configparser -import os.path -import re -from importlib import import_module - -from radicale import storage -from radicale.log import logger - -INTERNAL_TYPES = ("none", "authenticated", "owner_write", "owner_only", - "from_file") - - -def load(configuration): - """Load the rights manager chosen in configuration.""" - rights_type = configuration.get("rights", "type") - if rights_type == "authenticated": - rights_class = AuthenticatedRights - elif rights_type == "owner_write": - rights_class = OwnerWriteRights - elif rights_type == "owner_only": - rights_class = OwnerOnlyRights - elif rights_type == "from_file": - rights_class = Rights - else: - try: - rights_class = import_module(rights_type).Rights - except Exception as e: - raise RuntimeError("Failed to load rights module %r: %s" % - (rights_type, e)) from e - logger.info("Rights type is %r", rights_type) - return rights_class(configuration) - - -def intersect_permissions(a, b="RrWw"): - return "".join(set(a).intersection(set(b))) - - -class BaseRights: - def __init__(self, configuration): - self.configuration = configuration - - def authorized(self, user, path, permissions): - """Check if the user is allowed to read or write the collection. - - If ``user`` is empty, check for anonymous rights. - - ``path`` is sanitized. - - ``permissions`` can include "R", "r", "W", "w" - - Returns granted rights. - - """ - raise NotImplementedError - - -class AuthenticatedRights(BaseRights): - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) - self._verify_user = self.configuration.get("auth", "type") != "none" - - def authorized(self, user, path, permissions): - if self._verify_user and not user: - return "" - sane_path = storage.sanitize_path(path).strip("/") - if "/" not in sane_path: - return intersect_permissions(permissions, "RW") - if sane_path.count("/") == 1: - return intersect_permissions(permissions, "rw") - return "" - - -class OwnerWriteRights(AuthenticatedRights): - def authorized(self, user, path, permissions): - if self._verify_user and not user: - return "" - sane_path = storage.sanitize_path(path).strip("/") - if not sane_path: - return intersect_permissions(permissions, "R") - if self._verify_user: - owned = user == sane_path.split("/", maxsplit=1)[0] - else: - owned = True - if "/" not in sane_path: - return intersect_permissions(permissions, "RW" if owned else "R") - if sane_path.count("/") == 1: - return intersect_permissions(permissions, "rw" if owned else "r") - return "" - - -class OwnerOnlyRights(AuthenticatedRights): - def authorized(self, user, path, permissions): - if self._verify_user and not user: - return "" - sane_path = storage.sanitize_path(path).strip("/") - if not sane_path: - return intersect_permissions(permissions, "R") - if self._verify_user and user != sane_path.split("/", maxsplit=1)[0]: - return "" - if "/" not in sane_path: - return intersect_permissions(permissions, "RW") - if sane_path.count("/") == 1: - return intersect_permissions(permissions, "rw") - return "" - - -class Rights(BaseRights): - def __init__(self, configuration): - super().__init__(configuration) - self.filename = os.path.expanduser(configuration.get("rights", "file")) - - def authorized(self, user, path, permissions): - user = user or "" - sane_path = storage.sanitize_path(path).strip("/") - # Prevent "regex injection" - user_escaped = re.escape(user) - sane_path_escaped = re.escape(sane_path) - rights_config = configparser.ConfigParser( - {"login": user_escaped, "path": sane_path_escaped}) - try: - if not rights_config.read(self.filename): - raise RuntimeError("No such file: %r" % - self.filename) - except Exception as e: - raise RuntimeError("Failed to load rights file %r: %s" % - (self.filename, e)) from e - for section in rights_config.sections(): - try: - user_pattern = rights_config.get(section, "user") - collection_pattern = rights_config.get(section, "collection") - user_match = re.fullmatch(user_pattern, user) - collection_match = user_match and re.fullmatch( - collection_pattern.format( - *map(re.escape, user_match.groups())), sane_path) - except Exception as e: - raise RuntimeError("Error in section %r of rights file %r: " - "%s" % (section, self.filename, e)) from e - if user_match and collection_match: - logger.debug("Rule %r:%r matches %r:%r from section %r", - user, sane_path, user_pattern, - collection_pattern, section) - return intersect_permissions( - permissions, rights_config.get(section, "permissions")) - else: - logger.debug("Rule %r:%r doesn't match %r:%r from section %r", - user, sane_path, user_pattern, - collection_pattern, section) - logger.info("Rights: %r:%r doesn't match any section", user, sane_path) - return "" diff --git a/radicale/rights/__init__.py b/radicale/rights/__init__.py new file mode 100644 index 00000000..b770a4d2 --- /dev/null +++ b/radicale/rights/__init__.py @@ -0,0 +1,84 @@ +# This file is part of Radicale Server - Calendar Server +# Copyright © 2012-2017 Guillaume Ayoub +# Copyright © 2017-2018 Unrud +# +# This library is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Radicale. If not, see . + +""" +Rights backends. + +This module loads the rights backend, according to the rights +configuration. + +Default rights are based on a regex-based file whose name is specified in the +config (section "right", key "file"). + +Authentication login is matched against the "user" key, and collection's path +is matched against the "collection" key. You can use Python's ConfigParser +interpolation values %(login)s and %(path)s. You can also get groups from the +user regex in the collection with {0}, {1}, etc. + +For example, for the "user" key, ".+" means "authenticated user" and ".*" +means "anybody" (including anonymous users). + +Section names are only used for naming the rule. + +Leading or ending slashes are trimmed from collection's path. + +""" + +from importlib import import_module + +from radicale.log import logger + +INTERNAL_TYPES = ("authenticated", "owner_write", "owner_only", "from_file") + + +def load(configuration): + """Load the rights manager chosen in configuration.""" + rights_type = configuration.get("rights", "type") + if rights_type in INTERNAL_TYPES: + module = "radicale.rights.%s" % rights_type + else: + module = rights_type + try: + class_ = import_module(module).Rights + except Exception as e: + raise RuntimeError("Failed to load rights module %r: %s" % + (rights_type, e)) from e + logger.info("Rights type is %r", rights_type) + return class_(configuration) + + +def intersect_permissions(a, b="RrWw"): + return "".join(set(a).intersection(set(b))) + + +class BaseRights: + def __init__(self, configuration): + self.configuration = configuration + + def authorized(self, user, path, permissions): + """Check if the user is allowed to read or write the collection. + + If ``user`` is empty, check for anonymous rights. + + ``path`` is sanitized. + + ``permissions`` can include "R", "r", "W", "w" + + Returns granted rights. + + """ + raise NotImplementedError diff --git a/radicale/rights/authenticated.py b/radicale/rights/authenticated.py new file mode 100644 index 00000000..fa013c9c --- /dev/null +++ b/radicale/rights/authenticated.py @@ -0,0 +1,35 @@ +# This file is part of Radicale Server - Calendar Server +# Copyright © 2012-2017 Guillaume Ayoub +# Copyright © 2017-2018 Unrud +# +# This library is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Radicale. If not, see . + + +from radicale import pathutils, rights + + +class Rights(rights.BaseRights): + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self._verify_user = self.configuration.get("auth", "type") != "none" + + def authorized(self, user, path, permissions): + if self._verify_user and not user: + return "" + sane_path = pathutils.sanitize_path(path).strip("/") + if "/" not in sane_path: + return rights.intersect_permissions(permissions, "RW") + if sane_path.count("/") == 1: + return rights.intersect_permissions(permissions, "rw") + return "" diff --git a/radicale/rights/from_file.py b/radicale/rights/from_file.py new file mode 100644 index 00000000..11612c82 --- /dev/null +++ b/radicale/rights/from_file.py @@ -0,0 +1,68 @@ +# This file is part of Radicale Server - Calendar Server +# Copyright © 2012-2017 Guillaume Ayoub +# Copyright © 2017-2018 Unrud +# +# This library is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Radicale. If not, see . + +import configparser +import os.path +import re + +from radicale import pathutils, rights +from radicale.log import logger + + +class Rights(rights.BaseRights): + def __init__(self, configuration): + super().__init__(configuration) + self.filename = os.path.expanduser(configuration.get("rights", "file")) + + def authorized(self, user, path, permissions): + user = user or "" + sane_path = pathutils.sanitize_path(path).strip("/") + # Prevent "regex injection" + user_escaped = re.escape(user) + sane_path_escaped = re.escape(sane_path) + rights_config = configparser.ConfigParser( + {"login": user_escaped, "path": sane_path_escaped}) + try: + if not rights_config.read(self.filename): + raise RuntimeError("No such file: %r" % + self.filename) + except Exception as e: + raise RuntimeError("Failed to load rights file %r: %s" % + (self.filename, e)) from e + for section in rights_config.sections(): + try: + user_pattern = rights_config.get(section, "user") + collection_pattern = rights_config.get(section, "collection") + user_match = re.fullmatch(user_pattern, user) + collection_match = user_match and re.fullmatch( + collection_pattern.format( + *map(re.escape, user_match.groups())), sane_path) + except Exception as e: + raise RuntimeError("Error in section %r of rights file %r: " + "%s" % (section, self.filename, e)) from e + if user_match and collection_match: + logger.debug("Rule %r:%r matches %r:%r from section %r", + user, sane_path, user_pattern, + collection_pattern, section) + return rights.intersect_permissions( + permissions, rights_config.get(section, "permissions")) + else: + logger.debug("Rule %r:%r doesn't match %r:%r from section %r", + user, sane_path, user_pattern, + collection_pattern, section) + logger.info("Rights: %r:%r doesn't match any section", user, sane_path) + return "" diff --git a/radicale/rights/owner_only.py b/radicale/rights/owner_only.py new file mode 100644 index 00000000..9c4e16b6 --- /dev/null +++ b/radicale/rights/owner_only.py @@ -0,0 +1,35 @@ +# This file is part of Radicale Server - Calendar Server +# Copyright © 2012-2017 Guillaume Ayoub +# Copyright © 2017-2018 Unrud +# +# This library is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Radicale. If not, see . + +import radicale.rights.authenticated as authenticated +from radicale import pathutils, rights + + +class Rights(authenticated.Rights): + def authorized(self, user, path, permissions): + if self._verify_user and not user: + return "" + sane_path = pathutils.sanitize_path(path).strip("/") + if not sane_path: + return rights.intersect_permissions(permissions, "R") + if self._verify_user and user != sane_path.split("/", maxsplit=1)[0]: + return "" + if "/" not in sane_path: + return rights.intersect_permissions(permissions, "RW") + if sane_path.count("/") == 1: + return rights.intersect_permissions(permissions, "rw") + return "" diff --git a/radicale/rights/owner_write.py b/radicale/rights/owner_write.py new file mode 100644 index 00000000..75709ee3 --- /dev/null +++ b/radicale/rights/owner_write.py @@ -0,0 +1,39 @@ +# This file is part of Radicale Server - Calendar Server +# Copyright © 2012-2017 Guillaume Ayoub +# Copyright © 2017-2018 Unrud +# +# This library is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Radicale. If not, see . + +import radicale.rights.authenticated as authenticated +from radicale import pathutils, rights + + +class Rights(authenticated.Rights): + def authorized(self, user, path, permissions): + if self._verify_user and not user: + return "" + sane_path = pathutils.sanitize_path(path).strip("/") + if not sane_path: + return rights.intersect_permissions(permissions, "R") + if self._verify_user: + owned = user == sane_path.split("/", maxsplit=1)[0] + else: + owned = True + if "/" not in sane_path: + return rights.intersect_permissions(permissions, + "RW" if owned else "R") + if sane_path.count("/") == 1: + return rights.intersect_permissions(permissions, + "rw" if owned else "r") + return "" diff --git a/radicale/server.py b/radicale/server.py index 700b5453..30a6848c 100644 --- a/radicale/server.py +++ b/radicale/server.py @@ -2,6 +2,7 @@ # Copyright © 2008 Nicolas Kandel # Copyright © 2008 Pascal Halter # Copyright © 2008-2017 Guillaume Ayoub +# Copyright © 2017-2018 Unrud # # This library is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by diff --git a/radicale/storage/__init__.py b/radicale/storage/__init__.py new file mode 100644 index 00000000..ef65fa26 --- /dev/null +++ b/radicale/storage/__init__.py @@ -0,0 +1,357 @@ +# This file is part of Radicale Server - Calendar Server +# Copyright © 2014 Jean-Marc Martins +# Copyright © 2012-2017 Guillaume Ayoub +# Copyright © 2017-2018 Unrud +# +# This library is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Radicale. If not, see . + +""" +Storage backends. + +This module loads the storage backend, according to the storage configuration. + +Default storage uses one folder per collection and one file per collection +entry. + +""" + +import json +from contextlib import contextmanager +from hashlib import md5 +from importlib import import_module + +import pkg_resources +import vobject + +from radicale.log import logger + +INTERNAL_TYPES = ("multifilesystem",) + +CACHE_DEPS = ("radicale", "vobject", "python-dateutil",) +CACHE_VERSION = (";".join(pkg_resources.get_distribution(pkg).version + for pkg in CACHE_DEPS) + ";").encode() + + +def load(configuration): + """Load the storage manager chosen in configuration.""" + storage_type = configuration.get("storage", "type") + if storage_type in INTERNAL_TYPES: + module = "radicale.storage.%s" % storage_type + else: + module = storage_type + try: + class_ = import_module(module).Collection + except Exception as e: + raise RuntimeError("Failed to load storage module %r: %s" % + (storage_type, e)) from e + logger.info("Storage type is %r", storage_type) + + class CollectionCopy(class_): + """Collection copy, avoids overriding the original class attributes.""" + CollectionCopy.configuration = configuration + CollectionCopy.static_init() + return CollectionCopy + + +class ComponentExistsError(ValueError): + def __init__(self, path): + message = "Component already exists: %r" % path + super().__init__(message) + + +class ComponentNotFoundError(ValueError): + def __init__(self, path): + message = "Component doesn't exist: %r" % path + super().__init__(message) + + +class BaseCollection: + + # Overriden on copy by the "load" function + configuration = None + + # Properties of instance + """The sanitized path of the collection without leading or trailing ``/``. + """ + path = "" + + @classmethod + def static_init(): + """init collection copy""" + pass + + @property + def owner(self): + """The owner of the collection.""" + return self.path.split("/", maxsplit=1)[0] + + @property + def is_principal(self): + """Collection is a principal.""" + return bool(self.path) and "/" not in self.path + + @classmethod + def discover(cls, path, depth="0"): + """Discover a list of collections under the given ``path``. + + ``path`` is sanitized. + + If ``depth`` is "0", only the actual object under ``path`` is + returned. + + If ``depth`` is anything but "0", it is considered as "1" and direct + children are included in the result. + + The root collection "/" must always exist. + + """ + raise NotImplementedError + + @classmethod + def move(cls, item, to_collection, to_href): + """Move an object. + + ``item`` is the item to move. + + ``to_collection`` is the target collection. + + ``to_href`` is the target name in ``to_collection``. An item with the + same name might already exist. + + """ + if item.collection.path == to_collection.path and item.href == to_href: + return + to_collection.upload(to_href, item) + item.collection.delete(item.href) + + @property + def etag(self): + """Encoded as quoted-string (see RFC 2616).""" + etag = md5() + for item in self.get_all(): + etag.update((item.href + "/" + item.etag).encode("utf-8")) + etag.update(json.dumps(self.get_meta(), sort_keys=True).encode()) + return '"%s"' % etag.hexdigest() + + @classmethod + def create_collection(cls, href, items=None, props=None): + """Create a collection. + + ``href`` is the sanitized path. + + If the collection already exists and neither ``collection`` nor + ``props`` are set, this method shouldn't do anything. Otherwise the + existing collection must be replaced. + + ``collection`` is a list of vobject components. + + ``props`` are metadata values for the collection. + + ``props["tag"]`` is the type of collection (VCALENDAR or + VADDRESSBOOK). If the key ``tag`` is missing, it is guessed from the + collection. + + """ + raise NotImplementedError + + def sync(self, old_token=None): + """Get the current sync token and changed items for synchronization. + + ``old_token`` an old sync token which is used as the base of the + delta update. If sync token is missing, all items are returned. + ValueError is raised for invalid or old tokens. + + WARNING: This simple default implementation treats all sync-token as + invalid. It adheres to the specification but some clients + (e.g. InfCloud) don't like it. Subclasses should provide a + more sophisticated implementation. + + """ + token = "http://radicale.org/ns/sync/%s" % self.etag.strip("\"") + if old_token: + raise ValueError("Sync token are not supported") + return token, self.list() + + def list(self): + """List collection items.""" + raise NotImplementedError + + def get(self, href): + """Fetch a single item.""" + raise NotImplementedError + + def get_multi(self, hrefs): + """Fetch multiple items. + + Functionally similar to ``get``, but might bring performance benefits + on some storages when used cleverly. It's not required to return the + requested items in the correct order. Duplicated hrefs can be ignored. + + Returns tuples with the href and the item or None if the item doesn't + exist. + + """ + return ((href, self.get(href)) for href in hrefs) + + def get_all(self): + """Fetch all items. + + Functionally similar to ``get``, but might bring performance benefits + on some storages when used cleverly. + + """ + return map(self.get, self.list()) + + def get_all_filtered(self, filters): + """Fetch all items with optional filtering. + + This can largely improve performance of reports depending on + the filters and this implementation. + + Returns tuples in the form ``(item, filters_matched)``. + ``filters_matched`` is a bool that indicates if ``filters`` are fully + matched. + + This returns all events by default + """ + return ((item, False) for item in self.get_all()) + + def has(self, href): + """Check if an item exists by its href. + + Functionally similar to ``get``, but might bring performance benefits + on some storages when used cleverly. + + """ + return self.get(href) is not None + + def has_uid(self, uid): + """Check if a UID exists in the collection.""" + for item in self.get_all(): + if item.uid == uid: + return True + return False + + def upload(self, href, item): + """Upload a new or replace an existing item.""" + raise NotImplementedError + + def delete(self, href=None): + """Delete an item. + + When ``href`` is ``None``, delete the collection. + + """ + raise NotImplementedError + + def get_meta(self, key=None): + """Get metadata value for collection. + + Return the value of the property ``key``. If ``key`` is ``None`` return + a dict with all properties + + """ + raise NotImplementedError + + def set_meta(self, props): + """Set metadata values for collection. + + ``props`` a dict with values for properties. + + """ + raise NotImplementedError + + @property + def last_modified(self): + """Get the HTTP-datetime of when the collection was modified.""" + raise NotImplementedError + + def serialize(self): + """Get the unicode string representing the whole collection.""" + if self.get_meta("tag") == "VCALENDAR": + in_vcalendar = False + vtimezones = "" + included_tzids = set() + vtimezone = [] + tzid = None + components = "" + # Concatenate all child elements of VCALENDAR from all items + # together, while preventing duplicated VTIMEZONE entries. + # VTIMEZONEs are only distinguished by their TZID, if different + # timezones share the same TZID this produces errornous ouput. + # VObject fails at this too. + for item in self.get_all(): + depth = 0 + for line in item.serialize().split("\r\n"): + if line.startswith("BEGIN:"): + depth += 1 + if depth == 1 and line == "BEGIN:VCALENDAR": + in_vcalendar = True + elif in_vcalendar: + if depth == 1 and line.startswith("END:"): + in_vcalendar = False + if depth == 2 and line == "BEGIN:VTIMEZONE": + vtimezone.append(line + "\r\n") + elif vtimezone: + vtimezone.append(line + "\r\n") + if depth == 2 and line.startswith("TZID:"): + tzid = line[len("TZID:"):] + elif depth == 2 and line.startswith("END:"): + if tzid is None or tzid not in included_tzids: + vtimezones += "".join(vtimezone) + included_tzids.add(tzid) + vtimezone.clear() + tzid = None + elif depth >= 2: + components += line + "\r\n" + if line.startswith("END:"): + depth -= 1 + template = vobject.iCalendar() + displayname = self.get_meta("D:displayname") + if displayname: + template.add("X-WR-CALNAME") + template.x_wr_calname.value_param = "TEXT" + template.x_wr_calname.value = displayname + description = self.get_meta("C:calendar-description") + if description: + template.add("X-WR-CALDESC") + template.x_wr_caldesc.value_param = "TEXT" + template.x_wr_caldesc.value = description + template = template.serialize() + template_insert_pos = template.find("\r\nEND:VCALENDAR\r\n") + 2 + assert template_insert_pos != -1 + return (template[:template_insert_pos] + + vtimezones + components + + template[template_insert_pos:]) + elif self.get_meta("tag") == "VADDRESSBOOK": + return "".join((item.serialize() for item in self.get_all())) + return "" + + @classmethod + @contextmanager + def acquire_lock(cls, mode, user=None): + """Set a context manager to lock the whole storage. + + ``mode`` must either be "r" for shared access or "w" for exclusive + access. + + ``user`` is the name of the logged in user or empty. + + """ + raise NotImplementedError + + @classmethod + def verify(cls): + """Check the storage for errors.""" + return True diff --git a/radicale/storage.py b/radicale/storage/multifilesystem.py similarity index 52% rename from radicale/storage.py rename to radicale/storage/multifilesystem.py index c48be0bd..995cb2f9 100644 --- a/radicale/storage.py +++ b/radicale/storage/multifilesystem.py @@ -1,6 +1,7 @@ # This file is part of Radicale Server - Calendar Server # Copyright © 2014 Jean-Marc Martins # Copyright © 2012-2017 Guillaume Ayoub +# Copyright © 2017-2018 Unrud # # This library is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -15,16 +16,6 @@ # You should have received a copy of the GNU General Public License # along with Radicale. If not, see . -""" -Storage backends. - -This module loads the storage backend, according to the storage configuration. - -Default storage uses one folder per collection and one file per collection -entry. - -""" - import binascii import contextlib import json @@ -34,753 +25,21 @@ import pickle import posixpath import shlex import subprocess -import threading import time from contextlib import contextmanager from hashlib import md5 -from importlib import import_module from itertools import chain -from random import getrandbits from tempfile import NamedTemporaryFile, TemporaryDirectory -import pkg_resources import vobject -from radicale import xmlutils +from radicale import item as radicale_item +from radicale import pathutils, storage +from radicale.item import filter as radicale_filter from radicale.log import logger -if os.name == "nt": - import ctypes - import ctypes.wintypes - import msvcrt - LOCKFILE_EXCLUSIVE_LOCK = 2 - if ctypes.sizeof(ctypes.c_void_p) == 4: - ULONG_PTR = ctypes.c_uint32 - else: - ULONG_PTR = ctypes.c_uint64 - - class Overlapped(ctypes.Structure): - _fields_ = [ - ("internal", ULONG_PTR), - ("internal_high", ULONG_PTR), - ("offset", ctypes.wintypes.DWORD), - ("offset_high", ctypes.wintypes.DWORD), - ("h_event", ctypes.wintypes.HANDLE)] - - lock_file_ex = ctypes.windll.kernel32.LockFileEx - lock_file_ex.argtypes = [ - ctypes.wintypes.HANDLE, - ctypes.wintypes.DWORD, - ctypes.wintypes.DWORD, - ctypes.wintypes.DWORD, - ctypes.wintypes.DWORD, - ctypes.POINTER(Overlapped)] - lock_file_ex.restype = ctypes.wintypes.BOOL - unlock_file_ex = ctypes.windll.kernel32.UnlockFileEx - unlock_file_ex.argtypes = [ - ctypes.wintypes.HANDLE, - ctypes.wintypes.DWORD, - ctypes.wintypes.DWORD, - ctypes.wintypes.DWORD, - ctypes.POINTER(Overlapped)] - unlock_file_ex.restype = ctypes.wintypes.BOOL -elif os.name == "posix": - import fcntl - -INTERNAL_TYPES = ("multifilesystem",) - -DEPS = ("radicale", "vobject", "python-dateutil",) -ITEM_CACHE_TAG = (";".join(pkg_resources.get_distribution(pkg).version - for pkg in DEPS) + ";").encode() - - -def load(configuration): - """Load the storage manager chosen in configuration.""" - storage_type = configuration.get("storage", "type") - if storage_type == "multifilesystem": - collection_class = Collection - else: - try: - collection_class = import_module(storage_type).Collection - except Exception as e: - raise RuntimeError("Failed to load storage module %r: %s" % - (storage_type, e)) from e - logger.info("Storage type is %r", storage_type) - - class CollectionCopy(collection_class): - """Collection copy, avoids overriding the original class attributes.""" - CollectionCopy.configuration = configuration - CollectionCopy.static_init() - return CollectionCopy - - -def predict_tag_of_parent_collection(vobject_items): - if len(vobject_items) != 1: - return "" - if vobject_items[0].name == "VCALENDAR": - return "VCALENDAR" - if vobject_items[0].name in ("VCARD", "VLIST"): - return "VADDRESSBOOK" - return "" - - -def predict_tag_of_whole_collection(vobject_items, fallback_tag=None): - if vobject_items and vobject_items[0].name == "VCALENDAR": - return "VCALENDAR" - if vobject_items and vobject_items[0].name in ("VCARD", "VLIST"): - return "VADDRESSBOOK" - if not fallback_tag and not vobject_items: - # Maybe an empty address book - return "VADDRESSBOOK" - return fallback_tag - - -def check_and_sanitize_items(vobject_items, is_collection=False, tag=None): - """Check vobject items for common errors and add missing UIDs. - - ``is_collection`` indicates that vobject_item contains unrelated - components. - - The ``tag`` of the collection. - - """ - if tag and tag not in ("VCALENDAR", "VADDRESSBOOK"): - raise ValueError("Unsupported collection tag: %r" % tag) - if not is_collection and len(vobject_items) != 1: - raise ValueError("Item contains %d components" % len(vobject_items)) - if tag == "VCALENDAR": - if len(vobject_items) > 1: - raise RuntimeError("VCALENDAR collection contains %d " - "components" % len(vobject_items)) - vobject_item = vobject_items[0] - if vobject_item.name != "VCALENDAR": - raise ValueError("Item type %r not supported in %r " - "collection" % (vobject_item.name, tag)) - component_uids = set() - for component in vobject_item.components(): - if component.name in ("VTODO", "VEVENT", "VJOURNAL"): - component_uid = get_uid(component) - if component_uid: - component_uids.add(component_uid) - component_name = None - object_uid = None - object_uid_set = False - for component in vobject_item.components(): - # https://tools.ietf.org/html/rfc4791#section-4.1 - if component.name == "VTIMEZONE": - continue - if component_name is None or is_collection: - component_name = component.name - elif component_name != component.name: - raise ValueError("Multiple component types in object: %r, %r" % - (component_name, component.name)) - if component_name not in ("VTODO", "VEVENT", "VJOURNAL"): - continue - component_uid = get_uid(component) - if not object_uid_set or is_collection: - object_uid_set = True - object_uid = component_uid - if not component_uid: - if not is_collection: - raise ValueError("%s component without UID in object" % - component_name) - component_uid = find_available_name( - component_uids.__contains__) - component_uids.add(component_uid) - if hasattr(component, "uid"): - component.uid.value = component_uid - else: - component.add("UID").value = component_uid - elif not object_uid or not component_uid: - raise ValueError("Multiple %s components without UID in " - "object" % component_name) - elif object_uid != component_uid: - raise ValueError( - "Multiple %s components with different UIDs in object: " - "%r, %r" % (component_name, object_uid, component_uid)) - # vobject interprets recurrence rules on demand - try: - component.rruleset - except Exception as e: - raise ValueError("invalid recurrence rules in %s" % - component.name) from e - elif tag == "VADDRESSBOOK": - # https://tools.ietf.org/html/rfc6352#section-5.1 - object_uids = set() - for vobject_item in vobject_items: - if vobject_item.name == "VCARD": - object_uid = get_uid(vobject_item) - if object_uid: - object_uids.add(object_uid) - for vobject_item in vobject_items: - if vobject_item.name == "VLIST": - # Custom format used by SOGo Connector to store lists of - # contacts - continue - if vobject_item.name != "VCARD": - raise ValueError("Item type %r not supported in %r " - "collection" % (vobject_item.name, tag)) - object_uid = get_uid(vobject_item) - if not object_uid: - if not is_collection: - raise ValueError("%s object without UID" % - vobject_item.name) - object_uid = find_available_name(object_uids.__contains__) - object_uids.add(object_uid) - if hasattr(vobject_item, "uid"): - vobject_item.uid.value = object_uid - else: - vobject_item.add("UID").value = object_uid - else: - for i in vobject_items: - raise ValueError("Item type %r not supported in %s collection" % - (i.name, repr(tag) if tag else "generic")) - - -def check_and_sanitize_props(props): - """Check collection properties for common errors.""" - tag = props.get("tag") - if tag and tag not in ("VCALENDAR", "VADDRESSBOOK"): - raise ValueError("Unsupported collection tag: %r" % tag) - - -def find_available_name(exists_fn, suffix=""): - """Generate a pseudo-random UID""" - # Prevent infinite loop - for _ in range(1000): - r = "%016x" % getrandbits(128) - name = "%s-%s-%s-%s-%s%s" % ( - r[:8], r[8:12], r[12:16], r[16:20], r[20:], suffix) - if not exists_fn(name): - return name - # something is wrong with the PRNG - raise RuntimeError("No unique random sequence found") - - -def get_etag(text): - """Etag from collection or item. - - Encoded as quoted-string (see RFC 2616). - - """ - etag = md5() - etag.update(text.encode("utf-8")) - return '"%s"' % etag.hexdigest() - - -def get_uid(vobject_component): - """UID value of an item if defined.""" - return (vobject_component.uid.value - if hasattr(vobject_component, "uid") else None) - - -def get_uid_from_object(vobject_item): - """UID value of an calendar/addressbook object.""" - if vobject_item.name == "VCALENDAR": - if hasattr(vobject_item, "vevent"): - return get_uid(vobject_item.vevent) - if hasattr(vobject_item, "vjournal"): - return get_uid(vobject_item.vjournal) - if hasattr(vobject_item, "vtodo"): - return get_uid(vobject_item.vtodo) - elif vobject_item.name == "VCARD": - return get_uid(vobject_item) - return None - - -def sanitize_path(path): - """Make path absolute with leading slash to prevent access to other data. - - Preserve a potential trailing slash. - - """ - trailing_slash = "/" if path.endswith("/") else "" - path = posixpath.normpath(path) - new_path = "/" - for part in path.split("/"): - if not is_safe_path_component(part): - continue - new_path = posixpath.join(new_path, part) - trailing_slash = "" if new_path.endswith("/") else trailing_slash - return new_path + trailing_slash - - -def is_safe_path_component(path): - """Check if path is a single component of a path. - - Check that the path is safe to join too. - - """ - return path and "/" not in path and path not in (".", "..") - - -def is_safe_filesystem_path_component(path): - """Check if path is a single component of a local and posix filesystem - path. - - Check that the path is safe to join too. - - """ - return ( - path and not os.path.splitdrive(path)[0] and - not os.path.split(path)[0] and path not in (os.curdir, os.pardir) and - not path.startswith(".") and not path.endswith("~") and - is_safe_path_component(path)) - - -def path_to_filesystem(root, *paths): - """Convert path to a local filesystem path relative to base_folder. - - `root` must be a secure filesystem path, it will be prepend to the path. - - Conversion of `paths` is done in a secure manner, or raises ``ValueError``. - - """ - paths = [sanitize_path(path).strip("/") for path in paths] - safe_path = root - for path in paths: - if not path: - continue - for part in path.split("/"): - if not is_safe_filesystem_path_component(part): - raise UnsafePathError(part) - safe_path_parent = safe_path - safe_path = os.path.join(safe_path, part) - # Check for conflicting files (e.g. case-insensitive file systems - # or short names on Windows file systems) - if (os.path.lexists(safe_path) and - part not in (e.name for e in - os.scandir(safe_path_parent))): - raise CollidingPathError(part) - return safe_path - - -class UnsafePathError(ValueError): - def __init__(self, path): - message = "Can't translate name safely to filesystem: %r" % path - super().__init__(message) - - -class CollidingPathError(ValueError): - def __init__(self, path): - message = "File name collision: %r" % path - super().__init__(message) - - -class ComponentExistsError(ValueError): - def __init__(self, path): - message = "Component already exists: %r" % path - super().__init__(message) - - -class ComponentNotFoundError(ValueError): - def __init__(self, path): - message = "Component doesn't exist: %r" % path - super().__init__(message) - - -class Item: - def __init__(self, collection_path=None, collection=None, - vobject_item=None, href=None, last_modified=None, text=None, - etag=None, uid=None, name=None, component_name=None, - time_range=None): - """Initialize an item. - - ``collection_path`` the path of the parent collection (optional if - ``collection`` is set). - - ``collection`` the parent collection (optional). - - ``href`` the href of the item. - - ``last_modified`` the HTTP-datetime of when the item was modified. - - ``text`` the text representation of the item (optional if - ``vobject_item`` is set). - - ``vobject_item`` the vobject item (optional if ``text`` is set). - - ``etag`` the etag of the item (optional). See ``get_etag``. - - ``uid`` the UID of the object (optional). See ``get_uid_from_object``. - - ``name`` the name of the item (optional). See ``vobject_item.name``. - - ``component_name`` the name of the primary component (optional). - See ``find_tag``. - - ``time_range`` the enclosing time range. - See ``find_tag_and_time_range``. - - """ - if text is None and vobject_item is None: - raise ValueError( - "at least one of 'text' or 'vobject_item' must be set") - if collection_path is None: - if collection is None: - raise ValueError("at least one of 'collection_path' or " - "'collection' must be set") - collection_path = collection.path - self._collection_path = collection_path - self.collection = collection - self.href = href - self.last_modified = last_modified - self._text = text - self._vobject_item = vobject_item - self._etag = etag - self._uid = uid - self._name = name - self._component_name = component_name - self._time_range = time_range - - def serialize(self): - if self._text is None: - try: - self._text = self.vobject_item.serialize() - except Exception as e: - raise RuntimeError("Failed to serialize item %r from %r: %s" % - (self.href, self._collection_path, - e)) from e - return self._text - - @property - def vobject_item(self): - if self._vobject_item is None: - try: - self._vobject_item = vobject.readOne(self._text) - except Exception as e: - raise RuntimeError("Failed to parse item %r from %r: %s" % - (self.href, self._collection_path, - e)) from e - return self._vobject_item - - @property - def etag(self): - """Encoded as quoted-string (see RFC 2616).""" - if self._etag is None: - self._etag = get_etag(self.serialize()) - return self._etag - - @property - def uid(self): - if self._uid is None: - self._uid = get_uid_from_object(self.vobject_item) - return self._uid - - @property - def name(self): - if self._name is None: - self._name = self.vobject_item.name or "" - return self._name - - @property - def component_name(self): - if self._component_name is not None: - return self._component_name - return xmlutils.find_tag(self.vobject_item) - - @property - def time_range(self): - if self._time_range is None: - self._component_name, *self._time_range = ( - xmlutils.find_tag_and_time_range(self.vobject_item)) - return self._time_range - - def prepare(self): - """Fill cache with values.""" - orig_vobject_item = self._vobject_item - self.serialize() - self.etag - self.uid - self.name - self.time_range - self.component_name - self._vobject_item = orig_vobject_item - - -class BaseCollection: - - # Overriden on copy by the "load" function - configuration = None - - # Properties of instance - """The sanitized path of the collection without leading or trailing ``/``. - """ - path = "" - - @classmethod - def static_init(): - """init collection copy""" - pass - - @property - def owner(self): - """The owner of the collection.""" - return self.path.split("/", maxsplit=1)[0] - - @property - def is_principal(self): - """Collection is a principal.""" - return bool(self.path) and "/" not in self.path - - @classmethod - def discover(cls, path, depth="0"): - """Discover a list of collections under the given ``path``. - - ``path`` is sanitized. - - If ``depth`` is "0", only the actual object under ``path`` is - returned. - - If ``depth`` is anything but "0", it is considered as "1" and direct - children are included in the result. - - The root collection "/" must always exist. - - """ - raise NotImplementedError - - @classmethod - def move(cls, item, to_collection, to_href): - """Move an object. - - ``item`` is the item to move. - - ``to_collection`` is the target collection. - - ``to_href`` is the target name in ``to_collection``. An item with the - same name might already exist. - - """ - if item.collection.path == to_collection.path and item.href == to_href: - return - to_collection.upload(to_href, item) - item.collection.delete(item.href) - - @property - def etag(self): - """Encoded as quoted-string (see RFC 2616).""" - etag = md5() - for item in self.get_all(): - etag.update((item.href + "/" + item.etag).encode("utf-8")) - etag.update(json.dumps(self.get_meta(), sort_keys=True).encode()) - return '"%s"' % etag.hexdigest() - - @classmethod - def create_collection(cls, href, items=None, props=None): - """Create a collection. - - ``href`` is the sanitized path. - - If the collection already exists and neither ``collection`` nor - ``props`` are set, this method shouldn't do anything. Otherwise the - existing collection must be replaced. - - ``collection`` is a list of vobject components. - - ``props`` are metadata values for the collection. - - ``props["tag"]`` is the type of collection (VCALENDAR or - VADDRESSBOOK). If the key ``tag`` is missing, it is guessed from the - collection. - - """ - raise NotImplementedError - - def sync(self, old_token=None): - """Get the current sync token and changed items for synchronization. - - ``old_token`` an old sync token which is used as the base of the - delta update. If sync token is missing, all items are returned. - ValueError is raised for invalid or old tokens. - - WARNING: This simple default implementation treats all sync-token as - invalid. It adheres to the specification but some clients - (e.g. InfCloud) don't like it. Subclasses should provide a - more sophisticated implementation. - - """ - token = "http://radicale.org/ns/sync/%s" % self.etag.strip("\"") - if old_token: - raise ValueError("Sync token are not supported") - return token, self.list() - - def list(self): - """List collection items.""" - raise NotImplementedError - - def get(self, href): - """Fetch a single item.""" - raise NotImplementedError - - def get_multi(self, hrefs): - """Fetch multiple items. - - Functionally similar to ``get``, but might bring performance benefits - on some storages when used cleverly. It's not required to return the - requested items in the correct order. Duplicated hrefs can be ignored. - - Returns tuples with the href and the item or None if the item doesn't - exist. - - """ - return ((href, self.get(href)) for href in hrefs) - - def get_all(self): - """Fetch all items. - - Functionally similar to ``get``, but might bring performance benefits - on some storages when used cleverly. - - """ - return map(self.get, self.list()) - - def get_all_filtered(self, filters): - """Fetch all items with optional filtering. - - This can largely improve performance of reports depending on - the filters and this implementation. - - Returns tuples in the form ``(item, filters_matched)``. - ``filters_matched`` is a bool that indicates if ``filters`` are fully - matched. - - This returns all events by default - """ - return ((item, False) for item in self.get_all()) - - def has(self, href): - """Check if an item exists by its href. - - Functionally similar to ``get``, but might bring performance benefits - on some storages when used cleverly. - - """ - return self.get(href) is not None - - def has_uid(self, uid): - """Check if a UID exists in the collection.""" - for item in self.get_all(): - if item.uid == uid: - return True - return False - - def upload(self, href, item): - """Upload a new or replace an existing item.""" - raise NotImplementedError - - def delete(self, href=None): - """Delete an item. - - When ``href`` is ``None``, delete the collection. - - """ - raise NotImplementedError - - def get_meta(self, key=None): - """Get metadata value for collection. - - Return the value of the property ``key``. If ``key`` is ``None`` return - a dict with all properties - - """ - raise NotImplementedError - - def set_meta(self, props): - """Set metadata values for collection. - - ``props`` a dict with values for properties. - - """ - raise NotImplementedError - - @property - def last_modified(self): - """Get the HTTP-datetime of when the collection was modified.""" - raise NotImplementedError - - def serialize(self): - """Get the unicode string representing the whole collection.""" - if self.get_meta("tag") == "VCALENDAR": - in_vcalendar = False - vtimezones = "" - included_tzids = set() - vtimezone = [] - tzid = None - components = "" - # Concatenate all child elements of VCALENDAR from all items - # together, while preventing duplicated VTIMEZONE entries. - # VTIMEZONEs are only distinguished by their TZID, if different - # timezones share the same TZID this produces errornous ouput. - # VObject fails at this too. - for item in self.get_all(): - depth = 0 - for line in item.serialize().split("\r\n"): - if line.startswith("BEGIN:"): - depth += 1 - if depth == 1 and line == "BEGIN:VCALENDAR": - in_vcalendar = True - elif in_vcalendar: - if depth == 1 and line.startswith("END:"): - in_vcalendar = False - if depth == 2 and line == "BEGIN:VTIMEZONE": - vtimezone.append(line + "\r\n") - elif vtimezone: - vtimezone.append(line + "\r\n") - if depth == 2 and line.startswith("TZID:"): - tzid = line[len("TZID:"):] - elif depth == 2 and line.startswith("END:"): - if tzid is None or tzid not in included_tzids: - vtimezones += "".join(vtimezone) - included_tzids.add(tzid) - vtimezone.clear() - tzid = None - elif depth >= 2: - components += line + "\r\n" - if line.startswith("END:"): - depth -= 1 - template = vobject.iCalendar() - displayname = self.get_meta("D:displayname") - if displayname: - template.add("X-WR-CALNAME") - template.x_wr_calname.value_param = "TEXT" - template.x_wr_calname.value = displayname - description = self.get_meta("C:calendar-description") - if description: - template.add("X-WR-CALDESC") - template.x_wr_caldesc.value_param = "TEXT" - template.x_wr_caldesc.value = description - template = template.serialize() - template_insert_pos = template.find("\r\nEND:VCALENDAR\r\n") + 2 - assert template_insert_pos != -1 - return (template[:template_insert_pos] + - vtimezones + components + - template[template_insert_pos:]) - elif self.get_meta("tag") == "VADDRESSBOOK": - return "".join((item.serialize() for item in self.get_all())) - return "" - - @classmethod - @contextmanager - def acquire_lock(cls, mode, user=None): - """Set a context manager to lock the whole storage. - - ``mode`` must either be "r" for shared access or "w" for exclusive - access. - - ``user`` is the name of the logged in user or empty. - - """ - raise NotImplementedError - - @classmethod - def verify(cls): - """Check the storage for errors.""" - return True - - -class Collection(BaseCollection): +class Collection(storage.BaseCollection): """Collection stored in several files per calendar.""" @classmethod @@ -790,15 +49,15 @@ class Collection(BaseCollection): "storage", "filesystem_folder")) cls._makedirs_synced(folder) lock_path = os.path.join(folder, ".Radicale.lock") - cls._lock = FileBackedRwLock(lock_path) + cls._lock = pathutils.RwLock(lock_path) def __init__(self, path, filesystem_path=None): folder = self._get_collection_root_folder() # Path should already be sanitized - self.path = sanitize_path(path).strip("/") + self.path = pathutils.sanitize_path(path).strip("/") self._encoding = self.configuration.get("encoding", "stock") if filesystem_path is None: - filesystem_path = path_to_filesystem(folder, self.path) + filesystem_path = pathutils.path_to_filesystem(folder, self.path) self._filesystem_path = filesystem_path self._props_path = os.path.join( self._filesystem_path, ".Radicale.props") @@ -839,10 +98,7 @@ class Collection(BaseCollection): @classmethod def _fsync(cls, fd): if cls.configuration.getboolean("internal", "filesystem_fsync"): - if os.name == "posix" and hasattr(fcntl, "F_FULLFSYNC"): - fcntl.fcntl(fd, fcntl.F_FULLFSYNC) - else: - os.fsync(fd) + pathutils.fsync(fd) @classmethod def _sync_directory(cls, path): @@ -886,14 +142,14 @@ class Collection(BaseCollection): def discover(cls, path, depth="0", child_context_manager=( lambda path, href=None: contextlib.ExitStack())): # Path should already be sanitized - sane_path = sanitize_path(path).strip("/") + sane_path = pathutils.sanitize_path(path).strip("/") attributes = sane_path.split("/") if sane_path else [] folder = cls._get_collection_root_folder() # Create the root collection cls._makedirs_synced(folder) try: - filesystem_path = path_to_filesystem(folder, sane_path) + filesystem_path = pathutils.path_to_filesystem(folder, sane_path) except ValueError as e: # Path is unsafe logger.debug("Unsafe path %r requested from storage: %s", @@ -929,7 +185,7 @@ class Collection(BaseCollection): if not entry.is_dir(): continue href = entry.name - if not is_safe_filesystem_path_component(href): + if not pathutils.is_safe_filesystem_path_component(href): if not href.startswith(".Radicale"): logger.debug("Skipping collection %r in %r", href, sane_path) @@ -970,7 +226,7 @@ class Collection(BaseCollection): collection = item collection.get_meta() continue - if isinstance(item, BaseCollection): + if isinstance(item, storage.BaseCollection): has_child_collections = True remaining_paths.append(item.path) elif item.uid in uids: @@ -993,8 +249,8 @@ class Collection(BaseCollection): folder = cls._get_collection_root_folder() # Path should already be sanitized - sane_path = sanitize_path(href).strip("/") - filesystem_path = path_to_filesystem(folder, sane_path) + sane_path = pathutils.sanitize_path(href).strip("/") + filesystem_path = pathutils.path_to_filesystem(folder, sane_path) if not props: cls._makedirs_synced(filesystem_path) @@ -1052,8 +308,9 @@ class Collection(BaseCollection): lambda: uid if uid.lower().endswith(suffix.lower()) else uid + suffix) href_candidates.extend(( - lambda: get_etag(uid).strip('"') + suffix, - lambda: find_available_name(hrefs.__contains__, suffix))) + lambda: radicale_item.get_etag(uid).strip('"') + suffix, + lambda: radicale_item.find_available_uid(hrefs.__contains__, + suffix))) href = None def replace_fn(source, target): @@ -1062,12 +319,12 @@ class Collection(BaseCollection): href = href_candidates.pop(0)() if href in hrefs: continue - if not is_safe_filesystem_path_component(href): + if not pathutils.is_safe_filesystem_path_component(href): if not href_candidates: - raise UnsafePathError(href) + raise pathutils.UnsafePathError(href) continue try: - return os.replace(source, path_to_filesystem( + return os.replace(source, pathutils.path_to_filesystem( self._filesystem_path, href)) except OSError as e: if href_candidates and ( @@ -1089,11 +346,13 @@ class Collection(BaseCollection): @classmethod def move(cls, item, to_collection, to_href): - if not is_safe_filesystem_path_component(to_href): - raise UnsafePathError(to_href) + if not pathutils.is_safe_filesystem_path_component(to_href): + raise pathutils.UnsafePathError(to_href) os.replace( - path_to_filesystem(item.collection._filesystem_path, item.href), - path_to_filesystem(to_collection._filesystem_path, to_href)) + pathutils.path_to_filesystem( + item.collection._filesystem_path, item.href), + pathutils.path_to_filesystem( + to_collection._filesystem_path, to_href)) cls._sync_directory(to_collection._filesystem_path) if item.collection._filesystem_path != to_collection._filesystem_path: cls._sync_directory(item.collection._filesystem_path) @@ -1126,7 +385,7 @@ class Collection(BaseCollection): age_limit = time.time() - max_age if max_age is not None else None modified = False for name in names: - if not is_safe_filesystem_path_component(name): + if not pathutils.is_safe_filesystem_path_component(name): continue if age_limit is not None: try: @@ -1172,7 +431,8 @@ class Collection(BaseCollection): etag = item.etag if item else "" if etag != cache_etag: self._makedirs_synced(history_folder) - history_etag = get_etag(history_etag + "/" + etag).strip("\"") + history_etag = radicale_item.get_etag( + history_etag + "/" + etag).strip("\"") try: # Race: Other processes might have created and locked the file. with self._atomic_write(os.path.join(history_folder, href), @@ -1190,7 +450,7 @@ class Collection(BaseCollection): try: for entry in os.scandir(history_folder): href = entry.name - if not is_safe_filesystem_path_component(href): + if not pathutils.is_safe_filesystem_path_component(href): continue if os.path.isfile(os.path.join(self._filesystem_path, href)): continue @@ -1304,7 +564,7 @@ class Collection(BaseCollection): if not entry.is_file(): continue href = entry.name - if not is_safe_filesystem_path_component(href): + if not pathutils.is_safe_filesystem_path_component(href): if not href.startswith(".Radicale"): logger.debug("Skipping item %r in %r", href, self.path) continue @@ -1312,7 +572,7 @@ class Collection(BaseCollection): def _item_cache_hash(self, raw_text): _hash = md5() - _hash.update(ITEM_CACHE_TAG) + _hash.update(storage.CACHE_VERSION) _hash.update(raw_text) return _hash.hexdigest() @@ -1345,7 +605,7 @@ class Collection(BaseCollection): self._makedirs_synced(cache_folder) lock_path = os.path.join(cache_folder, ".Radicale.lock" + (".%s" % ns if ns else "")) - lock = FileBackedRwLock(lock_path) + lock = pathutils.RwLock(lock_path) return lock.acquire("w") def _load_item_cache(self, href, input_hash): @@ -1374,9 +634,10 @@ class Collection(BaseCollection): def get(self, href, verify_href=True): if verify_href: try: - if not is_safe_filesystem_path_component(href): - raise UnsafePathError(href) - path = path_to_filesystem(self._filesystem_path, href) + if not pathutils.is_safe_filesystem_path_component(href): + raise pathutils.UnsafePathError(href) + path = pathutils.path_to_filesystem( + self._filesystem_path, href) except ValueError as e: logger.debug( "Can't translate name %r safely to filesystem in %r: %s", @@ -1413,11 +674,11 @@ class Collection(BaseCollection): try: vobject_items = tuple(vobject.readComponents( raw_text.decode(self._encoding))) - check_and_sanitize_items(vobject_items, - tag=self.get_meta("tag")) + radicale_item.check_and_sanitize_items( + vobject_items, tag=self.get_meta("tag")) vobject_item, = vobject_items - temp_item = Item(collection=self, - vobject_item=vobject_item) + temp_item = radicale_item.Item( + collection=self, vobject_item=vobject_item) cache_hash, uid, etag, text, name, tag, start, end = \ self._store_item_cache( href, temp_item, input_hash) @@ -1434,7 +695,7 @@ class Collection(BaseCollection): time.gmtime(os.path.getmtime(path))) # Don't keep reference to ``vobject_item``, because it requires a lot # of memory. - return Item( + return radicale_item.Item( collection=self, href=href, last_modified=last_modified, etag=etag, text=text, uid=uid, name=name, component_name=tag, time_range=(start, end)) @@ -1449,7 +710,7 @@ class Collection(BaseCollection): # empty and the for-loop is never executed. files = os.listdir(self._filesystem_path) path = os.path.join(self._filesystem_path, href) - if (not is_safe_filesystem_path_component(href) or + if (not pathutils.is_safe_filesystem_path_component(href) or href not in files and os.path.lexists(path)): logger.debug( "Can't translate name safely to filesystem: %r", href) @@ -1463,7 +724,7 @@ class Collection(BaseCollection): return (self.get(href, verify_href=False) for href in self.list()) def get_all_filtered(self, filters): - tag, start, end, simple = xmlutils.simplify_prefilters( + tag, start, end, simple = radicale_filter.simplify_prefilters( filters, collection_tag=self.get_meta("tag")) if not tag: # no filter @@ -1475,14 +736,14 @@ class Collection(BaseCollection): yield item, simple and (start <= istart or iend <= end) def upload(self, href, item): - if not is_safe_filesystem_path_component(href): - raise UnsafePathError(href) + if not pathutils.is_safe_filesystem_path_component(href): + raise pathutils.UnsafePathError(href) try: self._store_item_cache(href, item) except Exception as e: raise ValueError("Failed to store item %r in collection %r: %s" % (href, self.path, e)) from e - path = path_to_filesystem(self._filesystem_path, href) + path = pathutils.path_to_filesystem(self._filesystem_path, href) with self._atomic_write(path, newline="") as fd: fd.write(item.serialize()) # Clean the cache after the actual item is stored, or the cache entry @@ -1509,11 +770,11 @@ class Collection(BaseCollection): self._sync_directory(parent_dir) else: # Delete an item - if not is_safe_filesystem_path_component(href): - raise UnsafePathError(href) - path = path_to_filesystem(self._filesystem_path, href) + if not pathutils.is_safe_filesystem_path_component(href): + raise pathutils.UnsafePathError(href) + path = pathutils.path_to_filesystem(self._filesystem_path, href) if not os.path.isfile(path): - raise ComponentNotFoundError(href) + raise storage.ComponentNotFoundError(href) os.remove(path) self._sync_directory(os.path.dirname(path)) # Track the change @@ -1529,7 +790,7 @@ class Collection(BaseCollection): self._meta_cache = json.load(f) except FileNotFoundError: self._meta_cache = {} - check_and_sanitize_props(self._meta_cache) + radicale_item.check_and_sanitize_props(self._meta_cache) except ValueError as e: raise RuntimeError("Failed to load properties of collection " "%r: %s" % (self.path, e)) from e @@ -1580,60 +841,3 @@ class Collection(BaseCollection): logger.debug("Captured stderr hook:\n%s", stderr_data) if p.returncode != 0: raise subprocess.CalledProcessError(p.returncode, p.args) - - -class FileBackedRwLock: - """A readers-Writer lock that locks a file.""" - - def __init__(self, path): - self._path = path - self._readers = 0 - self._writer = False - self._lock = threading.Lock() - - @property - def locked(self): - with self._lock: - if self._readers > 0: - return "r" - if self._writer: - return "w" - return "" - - @contextmanager - def acquire(self, mode): - if mode not in "rw": - raise ValueError("Invalid mode: %r" % mode) - with open(self._path, "w+") as lock_file: - if os.name == "nt": - handle = msvcrt.get_osfhandle(lock_file.fileno()) - flags = LOCKFILE_EXCLUSIVE_LOCK if mode == "w" else 0 - overlapped = Overlapped() - if not lock_file_ex(handle, flags, 0, 1, 0, overlapped): - raise RuntimeError("Locking the storage failed: %s" % - ctypes.FormatError()) - elif os.name == "posix": - _cmd = fcntl.LOCK_EX if mode == "w" else fcntl.LOCK_SH - try: - fcntl.flock(lock_file.fileno(), _cmd) - except OSError as e: - raise RuntimeError("Locking the storage failed: %s" % - e) from e - else: - raise RuntimeError("Locking the storage failed: " - "Unsupported operating system") - with self._lock: - if self._writer or mode == "w" and self._readers != 0: - raise RuntimeError("Locking the storage failed: " - "Guarantees failed") - if mode == "r": - self._readers += 1 - else: - self._writer = True - try: - yield - finally: - with self._lock: - if mode == "r": - self._readers -= 1 - self._writer = False diff --git a/radicale/tests/__init__.py b/radicale/tests/__init__.py index 36ce10b6..22aeab00 100644 --- a/radicale/tests/__init__.py +++ b/radicale/tests/__init__.py @@ -1,5 +1,6 @@ # This file is part of Radicale Server - Calendar Server # Copyright © 2012-2017 Guillaume Ayoub +# Copyright © 2017-2018 Unrud # # This library is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by diff --git a/radicale/tests/custom/auth.py b/radicale/tests/custom/auth.py index b5ced620..b0a11726 100644 --- a/radicale/tests/custom/auth.py +++ b/radicale/tests/custom/auth.py @@ -2,6 +2,7 @@ # Copyright © 2008 Nicolas Kandel # Copyright © 2008 Pascal Halter # Copyright © 2008-2017 Guillaume Ayoub +# Copyright © 2017-2018 Unrud # # This library is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by diff --git a/radicale/tests/custom/rights.py b/radicale/tests/custom/rights.py index 97600443..193090cf 100644 --- a/radicale/tests/custom/rights.py +++ b/radicale/tests/custom/rights.py @@ -1,5 +1,5 @@ # This file is part of Radicale Server - Calendar Server -# Copyright (C) 2017 Unrud +# Copyright © 2017-2018 Unrud # # This library is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by diff --git a/radicale/tests/custom/storage.py b/radicale/tests/custom/storage.py index 621fdc2b..c12d7fd5 100644 --- a/radicale/tests/custom/storage.py +++ b/radicale/tests/custom/storage.py @@ -1,5 +1,6 @@ # This file is part of Radicale Server - Calendar Server # Copyright © 2012-2017 Guillaume Ayoub +# Copyright © 2017-2018 Unrud # # This library is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -21,11 +22,11 @@ Copy of filesystem storage backend for testing """ -from radicale import storage +from radicale.storage import multifilesystem # TODO: make something more in this collection (and test it) -class Collection(storage.Collection): +class Collection(multifilesystem.Collection): """Collection stored in a folder.""" def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) diff --git a/radicale/tests/helpers.py b/radicale/tests/helpers.py index c3d6bfc3..f59411ae 100644 --- a/radicale/tests/helpers.py +++ b/radicale/tests/helpers.py @@ -2,6 +2,7 @@ # Copyright © 2008 Nicolas Kandel # Copyright © 2008 Pascal Halter # Copyright © 2008-2017 Guillaume Ayoub +# Copyright © 2017-2018 Unrud # # This library is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by diff --git a/radicale/tests/test_auth.py b/radicale/tests/test_auth.py index 80c883d5..223bdf6b 100644 --- a/radicale/tests/test_auth.py +++ b/radicale/tests/test_auth.py @@ -1,6 +1,7 @@ # This file is part of Radicale Server - Calendar Server # Copyright © 2012-2016 Jean-Marc Martins # Copyright © 2012-2017 Guillaume Ayoub +# Copyright © 2017-2018 Unrud # # This library is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by diff --git a/radicale/tests/test_base.py b/radicale/tests/test_base.py index 5933ec07..c861094e 100644 --- a/radicale/tests/test_base.py +++ b/radicale/tests/test_base.py @@ -1,5 +1,6 @@ # This file is part of Radicale Server - Calendar Server # Copyright © 2012-2017 Guillaume Ayoub +# Copyright © 2017-2018 Unrud # # This library is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by diff --git a/radicale/tests/test_rights.py b/radicale/tests/test_rights.py index 6fc1f609..7cc16d93 100644 --- a/radicale/tests/test_rights.py +++ b/radicale/tests/test_rights.py @@ -1,5 +1,5 @@ # This file is part of Radicale Server - Calendar Server -# Copyright (C) 2017 Unrud +# Copyright © 2017-2018 Unrud # # This library is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by diff --git a/radicale/web/__init__.py b/radicale/web/__init__.py new file mode 100644 index 00000000..eb76b4a8 --- /dev/null +++ b/radicale/web/__init__.py @@ -0,0 +1,54 @@ +# This file is part of Radicale Server - Calendar Server +# Copyright © 2017-2018 Unrud +# +# This library is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Radicale. If not, see . + +from importlib import import_module + +from radicale.log import logger + +INTERNAL_TYPES = ("none", "internal") + + +def load(configuration): + """Load the web module chosen in configuration.""" + web_type = configuration.get("web", "type") + if web_type in INTERNAL_TYPES: + module = "radicale.web.%s" % web_type + else: + module = web_type + try: + class_ = import_module(module).Web + except Exception as e: + raise RuntimeError("Failed to load web module %r: %s" % + (web_type, e)) from e + logger.info("Web type is %r", web_type) + return class_(configuration) + + +class BaseWeb: + def __init__(self, configuration): + self.configuration = configuration + + def get(self, environ, base_prefix, path, user): + """GET request. + + ``base_prefix`` is sanitized and never ends with "/". + + ``path`` is sanitized and always starts with "/.web" + + ``user`` is empty for anonymous users. + + """ + raise NotImplementedError diff --git a/radicale/web.py b/radicale/web/internal.py similarity index 61% rename from radicale/web.py rename to radicale/web/internal.py index 2f9602d8..edc12b13 100644 --- a/radicale/web.py +++ b/radicale/web/internal.py @@ -1,5 +1,5 @@ # This file is part of Radicale Server - Calendar Server -# Copyright (C) 2017 Unrud +# Copyright © 2017-2018 Unrud # # This library is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -18,17 +18,12 @@ import os import posixpath import time from http import client -from importlib import import_module import pkg_resources -from radicale import storage +from radicale import httputils, pathutils, web from radicale.log import logger -NOT_FOUND = ( - client.NOT_FOUND, (("Content-Type", "text/plain"),), - "The requested resource could not be found.") - MIMETYPES = { ".css": "text/css", ".eot": "application/vnd.ms-fontobject", @@ -45,63 +40,21 @@ MIMETYPES = { ".xml": "text/xml"} FALLBACK_MIMETYPE = "application/octet-stream" -INTERNAL_TYPES = ("none", "internal") - -def load(configuration): - """Load the web module chosen in configuration.""" - web_type = configuration.get("web", "type") - if web_type == "none": - web_class = NoneWeb - elif web_type == "internal": - web_class = Web - else: - try: - web_class = import_module(web_type).Web - except Exception as e: - raise RuntimeError("Failed to load web module %r: %s" % - (web_type, e)) from e - logger.info("Web type is %r", web_type) - return web_class(configuration) - - -class BaseWeb: - def __init__(self, configuration): - self.configuration = configuration - - def get(self, environ, base_prefix, path, user): - """GET request. - - ``base_prefix`` is sanitized and never ends with "/". - - ``path`` is sanitized and always starts with "/.web" - - ``user`` is empty for anonymous users. - - """ - raise NotImplementedError - - -class NoneWeb(BaseWeb): - def get(self, environ, base_prefix, path, user): - if path != "/.web": - return NOT_FOUND - return client.OK, {"Content-Type": "text/plain"}, "Radicale works!" - - -class Web(BaseWeb): +class Web(web.BaseWeb): def __init__(self, configuration): super().__init__(configuration) - self.folder = pkg_resources.resource_filename(__name__, "web") + self.folder = pkg_resources.resource_filename(__name__, + "internal_data") def get(self, environ, base_prefix, path, user): try: - filesystem_path = storage.path_to_filesystem( + filesystem_path = pathutils.path_to_filesystem( self.folder, path[len("/.web"):]) except ValueError as e: logger.debug("Web content with unsafe path %r requested: %s", path, e, exc_info=True) - return NOT_FOUND + return httputils.NOT_FOUND if os.path.isdir(filesystem_path) and not path.endswith("/"): location = posixpath.basename(path) + "/" return (client.FOUND, @@ -110,7 +63,7 @@ class Web(BaseWeb): if os.path.isdir(filesystem_path): filesystem_path = os.path.join(filesystem_path, "index.html") if not os.path.isfile(filesystem_path): - return NOT_FOUND + return httputils.NOT_FOUND content_type = MIMETYPES.get( os.path.splitext(filesystem_path)[1].lower(), FALLBACK_MIMETYPE) with open(filesystem_path, "rb") as f: diff --git a/radicale/web/css/icon.png b/radicale/web/internal_data/css/icon.png similarity index 100% rename from radicale/web/css/icon.png rename to radicale/web/internal_data/css/icon.png diff --git a/radicale/web/css/main.css b/radicale/web/internal_data/css/main.css similarity index 100% rename from radicale/web/css/main.css rename to radicale/web/internal_data/css/main.css diff --git a/radicale/web/fn.js b/radicale/web/internal_data/fn.js similarity index 99% rename from radicale/web/fn.js rename to radicale/web/internal_data/fn.js index 9dd6b9e3..9bec3536 100644 --- a/radicale/web/fn.js +++ b/radicale/web/internal_data/fn.js @@ -1,6 +1,6 @@ /** * This file is part of Radicale Server - Calendar Server - * Copyright (C) 2017 Unrud + * Copyright © 2017-2018 Unrud * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by diff --git a/radicale/web/index.html b/radicale/web/internal_data/index.html similarity index 100% rename from radicale/web/index.html rename to radicale/web/internal_data/index.html diff --git a/radicale/web/none.py b/radicale/web/none.py new file mode 100644 index 00000000..18ddaf7f --- /dev/null +++ b/radicale/web/none.py @@ -0,0 +1,26 @@ +# This file is part of Radicale Server - Calendar Server +# Copyright © 2017-2018 Unrud +# +# This library is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Radicale. If not, see . + +from http import client + +from radicale import httputils, web + + +class Web(web.BaseWeb): + def get(self, environ, base_prefix, path, user): + if path != "/.web": + return httputils.NOT_FOUND + return client.OK, {"Content-Type": "text/plain"}, "Radicale works!" diff --git a/radicale/xmlutils.py b/radicale/xmlutils.py index d7b9f5b6..bcc9aad6 100644 --- a/radicale/xmlutils.py +++ b/radicale/xmlutils.py @@ -2,6 +2,7 @@ # Copyright © 2008 Nicolas Kandel # Copyright © 2008 Pascal Halter # Copyright © 2008-2015 Guillaume Ayoub +# Copyright © 2017-2018 Unrud # # This library is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -26,19 +27,11 @@ in them for XML requests (all but PUT). """ import copy -import math -import posixpath import re -import sys import xml.etree.ElementTree as ET from collections import OrderedDict -from datetime import date, datetime, timedelta, timezone from http import client -from itertools import chain -from urllib.parse import quote, unquote, urlparse - -from radicale import storage -from radicale.log import logger +from urllib.parse import quote MIMETYPES = { "VADDRESSBOOK": "text/vcard", @@ -66,13 +59,6 @@ for short, url in NAMESPACES.items(): CLARK_TAG_REGEX = re.compile(r"{(?P[^}]*)}(?P.*)", re.VERBOSE) HUMAN_REGEX = re.compile(r"(?P[^:{}]*):(?P.*)", re.VERBOSE) -DAY = timedelta(days=1) -SECOND = timedelta(seconds=1) -DATETIME_MIN = datetime.min.replace(tzinfo=timezone.utc) -DATETIME_MAX = datetime.max.replace(tzinfo=timezone.utc) -TIMESTAMP_MIN = math.floor(DATETIME_MIN.timestamp()) -TIMESTAMP_MAX = math.ceil(DATETIME_MAX.timestamp()) - def pretty_xml(element, level=0): """Indent an ElementTree ``element`` and its children.""" @@ -95,12 +81,12 @@ def pretty_xml(element, level=0): return '\n%s' % ET.tostring(element, "unicode") -def _tag(short_name, local): +def make_tag(short_name, local): """Get XML Clark notation {uri(``short_name``)}``local``.""" return "{%s}%s" % (NAMESPACES[short_name], local) -def _tag_from_clark(name): +def tag_from_clark(name): """Get a human-readable variant of the XML Clark notation tag ``name``. For a given name using the XML Clark notation, return a human-readable @@ -117,526 +103,31 @@ def _tag_from_clark(name): return name -def _tag_from_human(name): +def tag_from_human(name): """Get an XML Clark notation tag from human-readable variant ``name``.""" match = HUMAN_REGEX.match(name) if match and match.group("namespace") in NAMESPACES: - return _tag(match.group("namespace"), match.group("tag")) + return make_tag(match.group("namespace"), match.group("tag")) return name -def _response(code): +def make_response(code): """Return full W3C names from HTTP status codes.""" return "HTTP/1.1 %i %s" % (code, client.responses[code]) -def _href(base_prefix, href): +def make_href(base_prefix, href): """Return prefixed href.""" return quote("%s%s" % (base_prefix, href)) def webdav_error(namespace, name): """Generate XML error message.""" - root = ET.Element(_tag("D", "error")) - root.append(ET.Element(_tag(namespace, name))) + root = ET.Element(make_tag("D", "error")) + root.append(ET.Element(make_tag(namespace, name))) return root -def _date_to_datetime(date_): - """Transform a date to a UTC datetime. - - If date_ is a datetime without timezone, return as UTC datetime. If date_ - is already a datetime with timezone, return as is. - - """ - if not isinstance(date_, datetime): - date_ = datetime.combine(date_, datetime.min.time()) - if not date_.tzinfo: - date_ = date_.replace(tzinfo=timezone.utc) - return date_ - - -def _comp_match(item, filter_, level=0): - """Check whether the ``item`` matches the comp ``filter_``. - - If ``level`` is ``0``, the filter is applied on the - item's collection. Otherwise, it's applied on the item. - - See rfc4791-9.7.1. - - """ - - # TODO: Filtering VALARM and VFREEBUSY is not implemented - # HACK: the filters are tested separately against all components - - if level == 0: - tag = item.name - elif level == 1: - tag = item.component_name - else: - logger.warning( - "Filters with three levels of comp-filter are not supported") - return True - if not tag: - return False - name = filter_.get("name").upper() - if len(filter_) == 0: - # Point #1 of rfc4791-9.7.1 - return name == tag - if len(filter_) == 1: - if filter_[0].tag == _tag("C", "is-not-defined"): - # Point #2 of rfc4791-9.7.1 - return name != tag - if name != tag: - return False - if (level == 0 and name != "VCALENDAR" or - level == 1 and name not in ("VTODO", "VEVENT", "VJOURNAL")): - logger.warning("Filtering %s is not supported" % name) - return True - # Point #3 and #4 of rfc4791-9.7.1 - components = ([item.vobject_item] if level == 0 - else list(getattr(item.vobject_item, - "%s_list" % tag.lower()))) - for child in filter_: - if child.tag == _tag("C", "prop-filter"): - if not any(_prop_match(comp, child, "C") - for comp in components): - return False - elif child.tag == _tag("C", "time-range"): - if not _time_range_match(item.vobject_item, filter_[0], tag): - return False - elif child.tag == _tag("C", "comp-filter"): - if not _comp_match(item, child, level=level + 1): - return False - else: - raise ValueError("Unexpected %r in comp-filter" % child.tag) - return True - - -def _prop_match(vobject_item, filter_, ns): - """Check whether the ``item`` matches the prop ``filter_``. - - See rfc4791-9.7.2 and rfc6352-10.5.1. - - """ - name = filter_.get("name").lower() - if len(filter_) == 0: - # Point #1 of rfc4791-9.7.2 - return name in vobject_item.contents - if len(filter_) == 1: - if filter_[0].tag == _tag("C", "is-not-defined"): - # Point #2 of rfc4791-9.7.2 - return name not in vobject_item.contents - if name not in vobject_item.contents: - return False - # Point #3 and #4 of rfc4791-9.7.2 - for child in filter_: - if ns == "C" and child.tag == _tag("C", "time-range"): - if not _time_range_match(vobject_item, child, name): - return False - elif child.tag == _tag(ns, "text-match"): - if not _text_match(vobject_item, child, name, ns): - return False - elif child.tag == _tag(ns, "param-filter"): - if not _param_filter_match(vobject_item, child, name, ns): - return False - else: - raise ValueError("Unexpected %r in prop-filter" % child.tag) - return True - - -def _time_range_match(vobject_item, filter_, child_name): - """Check whether the component/property ``child_name`` of - ``vobject_item`` matches the time-range ``filter_``.""" - - start = filter_.get("start") - end = filter_.get("end") - if not start and not end: - return False - if start: - start = datetime.strptime(start, "%Y%m%dT%H%M%SZ") - else: - start = datetime.min - if end: - end = datetime.strptime(end, "%Y%m%dT%H%M%SZ") - else: - end = datetime.max - start = start.replace(tzinfo=timezone.utc) - end = end.replace(tzinfo=timezone.utc) - - matched = False - - def range_fn(range_start, range_end, is_recurrence): - nonlocal matched - if start < range_end and range_start < end: - matched = True - return True - if end < range_start and not is_recurrence: - return True - return False - - def infinity_fn(start): - return False - - _visit_time_ranges(vobject_item, child_name, range_fn, infinity_fn) - return matched - - -def _visit_time_ranges(vobject_item, child_name, range_fn, infinity_fn): - """Visit all time ranges in the component/property ``child_name`` of - `vobject_item`` with visitors ``range_fn`` and ``infinity_fn``. - - ``range_fn`` gets called for every time_range with ``start`` and ``end`` - datetimes and ``is_recurrence`` as arguments. If the function returns True, - the operation is cancelled. - - ``infinity_fn`` gets called when an infiite recurrence rule is detected - with ``start`` datetime as argument. If the function returns True, the - operation is cancelled. - - See rfc4791-9.9. - - """ - - # HACK: According to rfc5545-3.8.4.4 an recurrance that is resheduled - # with Recurrence ID affects the recurrence itself and all following - # recurrences too. This is not respected and client don't seem to bother - # either. - - def getrruleset(child, ignore=()): - if (hasattr(child, "rrule") and - ";UNTIL=" not in child.rrule.value.upper() and - ";COUNT=" not in child.rrule.value.upper()): - for dtstart in child.getrruleset(addRDate=True): - if dtstart in ignore: - continue - if infinity_fn(_date_to_datetime(dtstart)): - return (), True - break - return filter(lambda dtstart: dtstart not in ignore, - child.getrruleset(addRDate=True)), False - - def get_children(components): - main = None - recurrences = [] - for comp in components: - if hasattr(comp, "recurrence_id") and comp.recurrence_id.value: - recurrences.append(comp.recurrence_id.value) - if comp.rruleset: - # Prevent possible infinite loop - raise ValueError("Overwritten recurrence with RRULESET") - yield comp, True, () - else: - if main is not None: - raise ValueError("Multiple main components") - main = comp - if main is None: - raise ValueError("Main component missing") - yield main, False, recurrences - - # Comments give the lines in the tables of the specification - if child_name == "VEVENT": - for child, is_recurrence, recurrences in get_children( - vobject_item.vevent_list): - # TODO: check if there's a timezone - dtstart = child.dtstart.value - - if child.rruleset: - dtstarts, infinity = getrruleset(child, recurrences) - if infinity: - return - else: - dtstarts = (dtstart,) - - dtend = getattr(child, "dtend", None) - if dtend is not None: - dtend = dtend.value - original_duration = (dtend - dtstart).total_seconds() - dtend = _date_to_datetime(dtend) - - duration = getattr(child, "duration", None) - if duration is not None: - original_duration = duration = duration.value - - for dtstart in dtstarts: - dtstart_is_datetime = isinstance(dtstart, datetime) - dtstart = _date_to_datetime(dtstart) - - if dtend is not None: - # Line 1 - dtend = dtstart + timedelta(seconds=original_duration) - if range_fn(dtstart, dtend, is_recurrence): - return - elif duration is not None: - if original_duration is None: - original_duration = duration.seconds - if duration.seconds > 0: - # Line 2 - if range_fn(dtstart, dtstart + duration, - is_recurrence): - return - else: - # Line 3 - if range_fn(dtstart, dtstart + SECOND, is_recurrence): - return - elif dtstart_is_datetime: - # Line 4 - if range_fn(dtstart, dtstart + SECOND, is_recurrence): - return - else: - # Line 5 - if range_fn(dtstart, dtstart + DAY, is_recurrence): - return - - elif child_name == "VTODO": - for child, is_recurrence, recurrences in get_children( - vobject_item.vtodo_list): - dtstart = getattr(child, "dtstart", None) - duration = getattr(child, "duration", None) - due = getattr(child, "due", None) - completed = getattr(child, "completed", None) - created = getattr(child, "created", None) - - if dtstart is not None: - dtstart = _date_to_datetime(dtstart.value) - if duration is not None: - duration = duration.value - if due is not None: - due = _date_to_datetime(due.value) - if dtstart is not None: - original_duration = (due - dtstart).total_seconds() - if completed is not None: - completed = _date_to_datetime(completed.value) - if created is not None: - created = _date_to_datetime(created.value) - original_duration = (completed - created).total_seconds() - elif created is not None: - created = _date_to_datetime(created.value) - - if child.rruleset: - reference_dates, infinity = getrruleset(child, recurrences) - if infinity: - return - else: - if dtstart is not None: - reference_dates = (dtstart,) - elif due is not None: - reference_dates = (due,) - elif completed is not None: - reference_dates = (completed,) - elif created is not None: - reference_dates = (created,) - else: - # Line 8 - if range_fn(DATETIME_MIN, DATETIME_MAX, is_recurrence): - return - reference_dates = () - - for reference_date in reference_dates: - reference_date = _date_to_datetime(reference_date) - - if dtstart is not None and duration is not None: - # Line 1 - if range_fn(reference_date, - reference_date + duration + SECOND, - is_recurrence): - return - if range_fn(reference_date + duration - SECOND, - reference_date + duration + SECOND, - is_recurrence): - return - elif dtstart is not None and due is not None: - # Line 2 - due = reference_date + timedelta(seconds=original_duration) - if (range_fn(reference_date, due, is_recurrence) or - range_fn(reference_date, - reference_date + SECOND, is_recurrence) or - range_fn(due - SECOND, due, is_recurrence) or - range_fn(due - SECOND, reference_date + SECOND, - is_recurrence)): - return - elif dtstart is not None: - if range_fn(reference_date, reference_date + SECOND, - is_recurrence): - return - elif due is not None: - # Line 4 - if range_fn(reference_date - SECOND, reference_date, - is_recurrence): - return - elif completed is not None and created is not None: - # Line 5 - completed = reference_date + timedelta( - seconds=original_duration) - if (range_fn(reference_date - SECOND, - reference_date + SECOND, - is_recurrence) or - range_fn(completed - SECOND, completed + SECOND, - is_recurrence) or - range_fn(reference_date - SECOND, - reference_date + SECOND, is_recurrence) or - range_fn(completed - SECOND, completed + SECOND, - is_recurrence)): - return - elif completed is not None: - # Line 6 - if range_fn(reference_date - SECOND, - reference_date + SECOND, is_recurrence): - return - elif created is not None: - # Line 7 - if range_fn(reference_date, DATETIME_MAX, is_recurrence): - return - - elif child_name == "VJOURNAL": - for child, is_recurrence, recurrences in get_children( - vobject_item.vjournal_list): - dtstart = getattr(child, "dtstart", None) - - if dtstart is not None: - dtstart = dtstart.value - if child.rruleset: - dtstarts, infinity = getrruleset(child, recurrences) - if infinity: - return - else: - dtstarts = (dtstart,) - - for dtstart in dtstarts: - dtstart_is_datetime = isinstance(dtstart, datetime) - dtstart = _date_to_datetime(dtstart) - - if dtstart_is_datetime: - # Line 1 - if range_fn(dtstart, dtstart + SECOND, is_recurrence): - return - else: - # Line 2 - if range_fn(dtstart, dtstart + DAY, is_recurrence): - return - - else: - # Match a property - child = getattr(vobject_item, child_name.lower()) - if isinstance(child, date): - range_fn(child, child + DAY, False) - elif isinstance(child, datetime): - range_fn(child, child + SECOND, False) - - -def _text_match(vobject_item, filter_, child_name, ns, attrib_name=None): - """Check whether the ``item`` matches the text-match ``filter_``. - - See rfc4791-9.7.5. - - """ - # TODO: collations are not supported, but the default ones needed - # for DAV servers are actually pretty useless. Texts are lowered to - # be case-insensitive, almost as the "i;ascii-casemap" value. - text = next(filter_.itertext()).lower() - match_type = "contains" - if ns == "CR": - match_type = filter_.get("match-type", match_type) - - def match(value): - value = value.lower() - if match_type == "equals": - return value == text - if match_type == "contains": - return text in value - if match_type == "starts-with": - return value.startswith(text) - if match_type == "ends-with": - return value.endswith(text) - raise ValueError("Unexpected text-match match-type: %r" % match_type) - - children = getattr(vobject_item, "%s_list" % child_name, []) - if attrib_name: - condition = any( - match(attrib) for child in children - for attrib in child.params.get(attrib_name, [])) - else: - condition = any(match(child.value) for child in children) - if filter_.get("negate-condition") == "yes": - return not condition - else: - return condition - - -def _param_filter_match(vobject_item, filter_, parent_name, ns): - """Check whether the ``item`` matches the param-filter ``filter_``. - - See rfc4791-9.7.3. - - """ - name = filter_.get("name").upper() - children = getattr(vobject_item, "%s_list" % parent_name, []) - condition = any(name in child.params for child in children) - if len(filter_): - if filter_[0].tag == _tag(ns, "text-match"): - return condition and _text_match( - vobject_item, filter_[0], parent_name, ns, name) - elif filter_[0].tag == _tag(ns, "is-not-defined"): - return not condition - else: - return condition - - -def simplify_prefilters(filters, collection_tag="VCALENDAR"): - """Creates a simplified condition from ``filters``. - - Returns a tuple (``tag``, ``start``, ``end``, ``simple``) where ``tag`` is - a string or None (match all) and ``start`` and ``end`` are POSIX - timestamps (as int). ``simple`` is a bool that indicates that ``filters`` - and the simplified condition are identical. - - """ - flat_filters = tuple(chain.from_iterable(filters)) - simple = len(flat_filters) <= 1 - for col_filter in flat_filters: - if collection_tag != "VCALENDAR": - simple = False - break - if (col_filter.tag != _tag("C", "comp-filter") or - col_filter.get("name").upper() != "VCALENDAR"): - simple = False - continue - simple &= len(col_filter) <= 1 - for comp_filter in col_filter: - if comp_filter.tag != _tag("C", "comp-filter"): - simple = False - continue - tag = comp_filter.get("name").upper() - if comp_filter.find(_tag("C", "is-not-defined")) is not None: - simple = False - continue - simple &= len(comp_filter) <= 1 - for time_filter in comp_filter: - if tag not in ("VTODO", "VEVENT", "VJOURNAL"): - simple = False - break - if time_filter.tag != _tag("C", "time-range"): - simple = False - continue - start = time_filter.get("start") - end = time_filter.get("end") - if start: - start = math.floor(datetime.strptime( - start, "%Y%m%dT%H%M%SZ").replace( - tzinfo=timezone.utc).timestamp()) - else: - start = TIMESTAMP_MIN - if end: - end = math.ceil(datetime.strptime( - end, "%Y%m%dT%H%M%SZ").replace( - tzinfo=timezone.utc).timestamp()) - else: - end = TIMESTAMP_MAX - return tag, start, end, simple - return tag, TIMESTAMP_MIN, TIMESTAMP_MAX, simple - return None, TIMESTAMP_MIN, TIMESTAMP_MAX, simple - - def get_content_type(item): """Get the content-type of an item with charset and component parameters. """ @@ -649,71 +140,6 @@ def get_content_type(item): return content_type -def find_tag(vobject_item): - """Find component name from ``vobject_item``.""" - if vobject_item.name == "VCALENDAR": - for component in vobject_item.components(): - if component.name != "VTIMEZONE": - return component.name or "" - return "" - - -def find_tag_and_time_range(vobject_item): - """Find component name and enclosing time range from ``vobject item``. - - Returns a tuple (``tag``, ``start``, ``end``) where ``tag`` is a string - and ``start`` and ``end`` are POSIX timestamps (as int). - - This is intened to be used for matching against simplified prefilters. - - """ - tag = find_tag(vobject_item) - if not tag: - return (tag, TIMESTAMP_MIN, TIMESTAMP_MAX) - start = end = None - - def range_fn(range_start, range_end, is_recurrence): - nonlocal start, end - if start is None or range_start < start: - start = range_start - if end is None or end < range_end: - end = range_end - return False - - def infinity_fn(range_start): - nonlocal start, end - if start is None or range_start < start: - start = range_start - end = DATETIME_MAX - return True - - _visit_time_ranges(vobject_item, tag, range_fn, infinity_fn) - if start is None: - start = DATETIME_MIN - if end is None: - end = DATETIME_MAX - try: - return tag, math.floor(start.timestamp()), math.ceil(end.timestamp()) - except ValueError as e: - if str(e) == ("offset must be a timedelta representing a whole " - "number of minutes") and sys.version_info < (3, 6): - raise RuntimeError("Unsupported in Python < 3.6: %s" % e) from e - raise - - -def name_from_path(path, collection): - """Return Radicale item name from ``path``.""" - path = path.strip("/") + "/" - start = collection.path + "/" - if not path.startswith(start): - raise ValueError("%r doesn't start with %r" % (path, start)) - name = path[len(start):][:-1] - if name and not storage.is_safe_path_component(name): - raise ValueError("%r is not a component in collection %r" % - (name, collection.path)) - return name - - def props_from_request(xml_request, actions=("set", "remove")): """Return a list of properties as a dictionary.""" result = OrderedDict() @@ -721,615 +147,29 @@ def props_from_request(xml_request, actions=("set", "remove")): return result for action in actions: - action_element = xml_request.find(_tag("D", action)) + action_element = xml_request.find(make_tag("D", action)) if action_element is not None: break else: action_element = xml_request - prop_element = action_element.find(_tag("D", "prop")) + prop_element = action_element.find(make_tag("D", "prop")) if prop_element is not None: for prop in prop_element: - if prop.tag == _tag("D", "resourcetype"): + if prop.tag == make_tag("D", "resourcetype"): for resource_type in prop: - if resource_type.tag == _tag("C", "calendar"): + if resource_type.tag == make_tag("C", "calendar"): result["tag"] = "VCALENDAR" break - elif resource_type.tag == _tag("CR", "addressbook"): + elif resource_type.tag == make_tag("CR", "addressbook"): result["tag"] = "VADDRESSBOOK" break - elif prop.tag == _tag("C", "supported-calendar-component-set"): - result[_tag_from_clark(prop.tag)] = ",".join( + elif prop.tag == make_tag("C", "supported-calendar-component-set"): + result[tag_from_clark(prop.tag)] = ",".join( supported_comp.attrib["name"] for supported_comp in prop - if supported_comp.tag == _tag("C", "comp")) + if supported_comp.tag == make_tag("C", "comp")) else: - result[_tag_from_clark(prop.tag)] = prop.text + result[tag_from_clark(prop.tag)] = prop.text return result - - -def delete(base_prefix, path, collection, href=None): - """Read and answer DELETE requests. - - Read rfc4918-9.6 for info. - - """ - collection.delete(href) - - multistatus = ET.Element(_tag("D", "multistatus")) - response = ET.Element(_tag("D", "response")) - multistatus.append(response) - - href = ET.Element(_tag("D", "href")) - href.text = _href(base_prefix, path) - response.append(href) - - status = ET.Element(_tag("D", "status")) - status.text = _response(200) - response.append(status) - - return multistatus - - -def propfind(base_prefix, path, xml_request, allowed_items, user): - """Read and answer PROPFIND requests. - - Read rfc4918-9.1 for info. - - The collections parameter is a list of collections that are to be included - in the output. - - """ - # A client may choose not to submit a request body. An empty PROPFIND - # request body MUST be treated as if it were an 'allprop' request. - top_tag = (xml_request[0] if xml_request is not None else - ET.Element(_tag("D", "allprop"))) - - props = () - allprop = False - propname = False - if top_tag.tag == _tag("D", "allprop"): - allprop = True - elif top_tag.tag == _tag("D", "propname"): - propname = True - elif top_tag.tag == _tag("D", "prop"): - props = [prop.tag for prop in top_tag] - - if _tag("D", "current-user-principal") in props and not user: - # Ask for authentication - # Returning the DAV:unauthenticated pseudo-principal as specified in - # RFC 5397 doesn't seem to work with DAVdroid. - return client.FORBIDDEN, None - - # Writing answer - multistatus = ET.Element(_tag("D", "multistatus")) - - for item, permission in allowed_items: - write = permission == "w" - response = _propfind_response( - base_prefix, path, item, props, user, write=write, - allprop=allprop, propname=propname) - if response: - multistatus.append(response) - - return client.MULTI_STATUS, multistatus - - -def _propfind_response(base_prefix, path, item, props, user, write=False, - propname=False, allprop=False): - """Build and return a PROPFIND response.""" - if propname and allprop or (props and (propname or allprop)): - raise ValueError("Only use one of props, propname and allprops") - is_collection = isinstance(item, storage.BaseCollection) - if is_collection: - is_leaf = item.get_meta("tag") in ("VADDRESSBOOK", "VCALENDAR") - collection = item - else: - collection = item.collection - - response = ET.Element(_tag("D", "response")) - - href = ET.Element(_tag("D", "href")) - if is_collection: - # Some clients expect collections to end with / - uri = "/%s/" % item.path if item.path else "/" - else: - uri = "/" + posixpath.join(collection.path, item.href) - - href.text = _href(base_prefix, uri) - response.append(href) - - propstat404 = ET.Element(_tag("D", "propstat")) - propstat200 = ET.Element(_tag("D", "propstat")) - response.append(propstat200) - - prop200 = ET.Element(_tag("D", "prop")) - propstat200.append(prop200) - - prop404 = ET.Element(_tag("D", "prop")) - propstat404.append(prop404) - - if propname or allprop: - props = [] - # Should list all properties that can be retrieved by the code below - props.append(_tag("D", "principal-collection-set")) - props.append(_tag("D", "current-user-principal")) - props.append(_tag("D", "current-user-privilege-set")) - props.append(_tag("D", "supported-report-set")) - props.append(_tag("D", "resourcetype")) - props.append(_tag("D", "owner")) - - if is_collection and collection.is_principal: - props.append(_tag("C", "calendar-user-address-set")) - props.append(_tag("D", "principal-URL")) - props.append(_tag("CR", "addressbook-home-set")) - props.append(_tag("C", "calendar-home-set")) - - if not is_collection or is_leaf: - props.append(_tag("D", "getetag")) - props.append(_tag("D", "getlastmodified")) - props.append(_tag("D", "getcontenttype")) - props.append(_tag("D", "getcontentlength")) - - if is_collection: - if is_leaf: - props.append(_tag("D", "displayname")) - props.append(_tag("D", "sync-token")) - if collection.get_meta("tag") == "VCALENDAR": - props.append(_tag("CS", "getctag")) - props.append(_tag("C", "supported-calendar-component-set")) - - meta = item.get_meta() - for tag in meta: - if tag == "tag": - continue - clark_tag = _tag_from_human(tag) - if clark_tag not in props: - props.append(clark_tag) - - if propname: - for tag in props: - prop200.append(ET.Element(tag)) - props = () - - for tag in props: - element = ET.Element(tag) - is404 = False - if tag == _tag("D", "getetag"): - if not is_collection or is_leaf: - element.text = item.etag - else: - is404 = True - elif tag == _tag("D", "getlastmodified"): - if not is_collection or is_leaf: - element.text = item.last_modified - else: - is404 = True - elif tag == _tag("D", "principal-collection-set"): - tag = ET.Element(_tag("D", "href")) - tag.text = _href(base_prefix, "/") - element.append(tag) - elif (tag in (_tag("C", "calendar-user-address-set"), - _tag("D", "principal-URL"), - _tag("CR", "addressbook-home-set"), - _tag("C", "calendar-home-set")) and - collection.is_principal and is_collection): - tag = ET.Element(_tag("D", "href")) - tag.text = _href(base_prefix, path) - element.append(tag) - elif tag == _tag("C", "supported-calendar-component-set"): - human_tag = _tag_from_clark(tag) - if is_collection and is_leaf: - meta = item.get_meta(human_tag) - if meta: - components = meta.split(",") - else: - components = ("VTODO", "VEVENT", "VJOURNAL") - for component in components: - comp = ET.Element(_tag("C", "comp")) - comp.set("name", component) - element.append(comp) - else: - is404 = True - elif tag == _tag("D", "current-user-principal"): - if user: - tag = ET.Element(_tag("D", "href")) - tag.text = _href(base_prefix, "/%s/" % user) - element.append(tag) - else: - element.append(ET.Element(_tag("D", "unauthenticated"))) - elif tag == _tag("D", "current-user-privilege-set"): - privileges = [("D", "read")] - if write: - privileges.append(("D", "all")) - privileges.append(("D", "write")) - privileges.append(("D", "write-properties")) - privileges.append(("D", "write-content")) - for ns, privilege_name in privileges: - privilege = ET.Element(_tag("D", "privilege")) - privilege.append(ET.Element(_tag(ns, privilege_name))) - element.append(privilege) - elif tag == _tag("D", "supported-report-set"): - # These 3 reports are not implemented - reports = [ - ("D", "expand-property"), - ("D", "principal-search-property-set"), - ("D", "principal-property-search")] - if is_collection and is_leaf: - reports.append(("D", "sync-collection")) - if item.get_meta("tag") == "VADDRESSBOOK": - reports.append(("CR", "addressbook-multiget")) - reports.append(("CR", "addressbook-query")) - elif item.get_meta("tag") == "VCALENDAR": - reports.append(("C", "calendar-multiget")) - reports.append(("C", "calendar-query")) - for ns, report_name in reports: - supported = ET.Element(_tag("D", "supported-report")) - report_tag = ET.Element(_tag("D", "report")) - supported_report_tag = ET.Element(_tag(ns, report_name)) - report_tag.append(supported_report_tag) - supported.append(report_tag) - element.append(supported) - elif tag == _tag("D", "getcontentlength"): - if not is_collection or is_leaf: - encoding = collection.configuration.get("encoding", "request") - element.text = str(len(item.serialize().encode(encoding))) - else: - is404 = True - elif tag == _tag("D", "owner"): - # return empty elment, if no owner available (rfc3744-5.1) - if collection.owner: - tag = ET.Element(_tag("D", "href")) - tag.text = _href(base_prefix, "/%s/" % collection.owner) - element.append(tag) - elif is_collection: - if tag == _tag("D", "getcontenttype"): - if is_leaf: - element.text = MIMETYPES[item.get_meta("tag")] - else: - is404 = True - elif tag == _tag("D", "resourcetype"): - if item.is_principal: - tag = ET.Element(_tag("D", "principal")) - element.append(tag) - if is_leaf: - if item.get_meta("tag") == "VADDRESSBOOK": - tag = ET.Element(_tag("CR", "addressbook")) - element.append(tag) - elif item.get_meta("tag") == "VCALENDAR": - tag = ET.Element(_tag("C", "calendar")) - element.append(tag) - tag = ET.Element(_tag("D", "collection")) - element.append(tag) - elif tag == _tag("RADICALE", "displayname"): - # Only for internal use by the web interface - displayname = item.get_meta("D:displayname") - if displayname is not None: - element.text = displayname - else: - is404 = True - elif tag == _tag("D", "displayname"): - displayname = item.get_meta("D:displayname") - if not displayname and is_leaf: - displayname = item.path - if displayname is not None: - element.text = displayname - else: - is404 = True - elif tag == _tag("CS", "getctag"): - if is_leaf: - element.text = item.etag - else: - is404 = True - elif tag == _tag("D", "sync-token"): - if is_leaf: - element.text, _ = item.sync() - else: - is404 = True - else: - human_tag = _tag_from_clark(tag) - meta = item.get_meta(human_tag) - if meta is not None: - element.text = meta - else: - is404 = True - # Not for collections - elif tag == _tag("D", "getcontenttype"): - element.text = get_content_type(item) - elif tag == _tag("D", "resourcetype"): - # resourcetype must be returned empty for non-collection elements - pass - else: - is404 = True - - if is404: - prop404.append(element) - else: - prop200.append(element) - - status200 = ET.Element(_tag("D", "status")) - status200.text = _response(200) - propstat200.append(status200) - - status404 = ET.Element(_tag("D", "status")) - status404.text = _response(404) - propstat404.append(status404) - if len(prop404): - response.append(propstat404) - - return response - - -def _add_propstat_to(element, tag, status_number): - """Add a PROPSTAT response structure to an element. - - The PROPSTAT answer structure is defined in rfc4918-9.1. It is added to the - given ``element``, for the following ``tag`` with the given - ``status_number``. - - """ - propstat = ET.Element(_tag("D", "propstat")) - element.append(propstat) - - prop = ET.Element(_tag("D", "prop")) - propstat.append(prop) - - clark_tag = tag if "{" in tag else _tag(*tag.split(":", 1)) - prop_tag = ET.Element(clark_tag) - prop.append(prop_tag) - - status = ET.Element(_tag("D", "status")) - status.text = _response(status_number) - propstat.append(status) - - -def proppatch(base_prefix, path, xml_request, collection): - """Read and answer PROPPATCH requests. - - Read rfc4918-9.2 for info. - - """ - props_to_set = props_from_request(xml_request, actions=("set",)) - props_to_remove = props_from_request(xml_request, actions=("remove",)) - - multistatus = ET.Element(_tag("D", "multistatus")) - response = ET.Element(_tag("D", "response")) - multistatus.append(response) - - href = ET.Element(_tag("D", "href")) - href.text = _href(base_prefix, path) - response.append(href) - - new_props = collection.get_meta() - for short_name, value in props_to_set.items(): - new_props[short_name] = value - _add_propstat_to(response, short_name, 200) - for short_name in props_to_remove: - try: - del new_props[short_name] - except KeyError: - pass - _add_propstat_to(response, short_name, 200) - storage.check_and_sanitize_props(new_props) - collection.set_meta(new_props) - - return multistatus - - -def report(base_prefix, path, xml_request, collection, unlock_storage_fn): - """Read and answer REPORT requests. - - Read rfc3253-3.6 for info. - - """ - multistatus = ET.Element(_tag("D", "multistatus")) - if xml_request is None: - return client.MULTI_STATUS, multistatus - root = xml_request - if root.tag in ( - _tag("D", "principal-search-property-set"), - _tag("D", "principal-property-search"), - _tag("D", "expand-property")): - # We don't support searching for principals or indirect retrieving of - # properties, just return an empty result. - # InfCloud asks for expand-property reports (even if we don't announce - # support for them) and stops working if an error code is returned. - logger.warning("Unsupported REPORT method %r on %r requested", - _tag_from_clark(root.tag), path) - return client.MULTI_STATUS, multistatus - if (root.tag == _tag("C", "calendar-multiget") and - collection.get_meta("tag") != "VCALENDAR" or - root.tag == _tag("CR", "addressbook-multiget") and - collection.get_meta("tag") != "VADDRESSBOOK" or - root.tag == _tag("D", "sync-collection") and - collection.get_meta("tag") not in ("VADDRESSBOOK", "VCALENDAR")): - logger.warning("Invalid REPORT method %r on %r requested", - _tag_from_clark(root.tag), path) - return (client.CONFLICT, - webdav_error("D", "supported-report")) - prop_element = root.find(_tag("D", "prop")) - props = ( - [prop.tag for prop in prop_element] - if prop_element is not None else []) - - if root.tag in ( - _tag("C", "calendar-multiget"), - _tag("CR", "addressbook-multiget")): - # Read rfc4791-7.9 for info - hreferences = set() - for href_element in root.findall(_tag("D", "href")): - href_path = storage.sanitize_path( - unquote(urlparse(href_element.text).path)) - if (href_path + "/").startswith(base_prefix + "/"): - hreferences.add(href_path[len(base_prefix):]) - else: - logger.warning("Skipping invalid path %r in REPORT request on " - "%r", href_path, path) - elif root.tag == _tag("D", "sync-collection"): - old_sync_token_element = root.find(_tag("D", "sync-token")) - old_sync_token = "" - if old_sync_token_element is not None and old_sync_token_element.text: - old_sync_token = old_sync_token_element.text.strip() - logger.debug("Client provided sync token: %r", old_sync_token) - try: - sync_token, names = collection.sync(old_sync_token) - except ValueError as e: - # Invalid sync token - logger.warning("Client provided invalid sync token %r: %s", - old_sync_token, e, exc_info=True) - return (client.CONFLICT, - webdav_error("D", "valid-sync-token")) - hreferences = ("/" + posixpath.join(collection.path, n) for n in names) - # Append current sync token to response - sync_token_element = ET.Element(_tag("D", "sync-token")) - sync_token_element.text = sync_token - multistatus.append(sync_token_element) - else: - hreferences = (path,) - filters = ( - root.findall("./%s" % _tag("C", "filter")) + - root.findall("./%s" % _tag("CR", "filter"))) - - def retrieve_items(collection, hreferences, multistatus): - """Retrieves all items that are referenced in ``hreferences`` from - ``collection`` and adds 404 responses for missing and invalid items - to ``multistatus``.""" - collection_requested = False - - def get_names(): - """Extracts all names from references in ``hreferences`` and adds - 404 responses for invalid references to ``multistatus``. - If the whole collections is referenced ``collection_requested`` - gets set to ``True``.""" - nonlocal collection_requested - for hreference in hreferences: - try: - name = name_from_path(hreference, collection) - except ValueError as e: - logger.warning("Skipping invalid path %r in REPORT request" - " on %r: %s", hreference, path, e) - response = _item_response(base_prefix, hreference, - found_item=False) - multistatus.append(response) - continue - if name: - # Reference is an item - yield name - else: - # Reference is a collection - collection_requested = True - - for name, item in collection.get_multi(get_names()): - if not item: - uri = "/" + posixpath.join(collection.path, name) - response = _item_response(base_prefix, uri, - found_item=False) - multistatus.append(response) - else: - yield item, False - if collection_requested: - yield from collection.get_all_filtered(filters) - - # Retrieve everything required for finishing the request. - retrieved_items = list(retrieve_items(collection, hreferences, - multistatus)) - collection_tag = collection.get_meta("tag") - # Don't access storage after this! - unlock_storage_fn() - - def match(item, filter_): - tag = collection_tag - if (tag == "VCALENDAR" and filter_.tag != _tag("C", filter_)): - if len(filter_) == 0: - return True - if len(filter_) > 1: - raise ValueError("Filter with %d children" % len(filter_)) - if filter_[0].tag != _tag("C", "comp-filter"): - raise ValueError("Unexpected %r in filter" % filter_[0].tag) - return _comp_match(item, filter_[0]) - if tag == "VADDRESSBOOK" and filter_.tag != _tag("CR", filter_): - for child in filter_: - if child.tag != _tag("CR", "prop-filter"): - raise ValueError("Unexpected %r in filter" % child.tag) - test = filter_.get("test", "anyof") - if test == "anyof": - return any(_prop_match(item.vobject_item, f, "CR") - for f in filter_) - if test == "allof": - return all(_prop_match(item.vobject_item, f, "CR") - for f in filter_) - raise ValueError("Unsupported filter test: %r" % test) - return all(_prop_match(item.vobject_item, f, "CR") - for f in filter_) - raise ValueError("unsupported filter %r for %r" % (filter_.tag, tag)) - - while retrieved_items: - # ``item.vobject_item`` might be accessed during filtering. - # Don't keep reference to ``item``, because VObject requires a lot of - # memory. - item, filters_matched = retrieved_items.pop(0) - if filters and not filters_matched: - try: - if not all(match(item, filter_) for filter_ in filters): - continue - except ValueError as e: - raise ValueError("Failed to filter item %r from %r: %s" % - (item.href, collection.path, e)) from e - except Exception as e: - raise RuntimeError("Failed to filter item %r from %r: %s" % - (item.href, collection.path, e)) from e - - found_props = [] - not_found_props = [] - - for tag in props: - element = ET.Element(tag) - if tag == _tag("D", "getetag"): - element.text = item.etag - found_props.append(element) - elif tag == _tag("D", "getcontenttype"): - element.text = get_content_type(item) - found_props.append(element) - elif tag in ( - _tag("C", "calendar-data"), - _tag("CR", "address-data")): - element.text = item.serialize() - found_props.append(element) - else: - not_found_props.append(element) - - uri = "/" + posixpath.join(collection.path, item.href) - multistatus.append(_item_response( - base_prefix, uri, found_props=found_props, - not_found_props=not_found_props, found_item=True)) - - return client.MULTI_STATUS, multistatus - - -def _item_response(base_prefix, href, found_props=(), not_found_props=(), - found_item=True): - response = ET.Element(_tag("D", "response")) - - href_tag = ET.Element(_tag("D", "href")) - href_tag.text = _href(base_prefix, href) - response.append(href_tag) - - if found_item: - for code, props in ((200, found_props), (404, not_found_props)): - if props: - propstat = ET.Element(_tag("D", "propstat")) - status = ET.Element(_tag("D", "status")) - status.text = _response(code) - prop_tag = ET.Element(_tag("D", "prop")) - for prop in props: - prop_tag.append(prop) - propstat.append(prop_tag) - propstat.append(status) - response.append(propstat) - else: - status = ET.Element(_tag("D", "status")) - status.text = _response(404) - response.append(status) - - return response diff --git a/setup.py b/setup.py index 3a25b0be..5316bcff 100644 --- a/setup.py +++ b/setup.py @@ -2,6 +2,7 @@ # # This file is part of Radicale Server - Calendar Server # Copyright © 2009-2017 Guillaume Ayoub +# Copyright © 2017-2018 Unrud # # This library is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -42,8 +43,10 @@ from setuptools import setup # When the version is updated, a new section in the NEWS.md file must be # added too. VERSION = "2.90.0" -WEB_FILES = ["web/css/icon.png", "web/css/main.css", "web/fn.js", - "web/index.html"] +WEB_FILES = ["web/internal_data/css/icon.png", + "web/internal_data/css/main.css", + "web/internal_data/fn.js", + "web/internal_data/index.html"] needs_pytest = {"pytest", "test", "ptr"}.intersection(sys.argv)