diff --git a/CHANGELOG.md b/CHANGELOG.md index a5c5a1de..87d3fd2b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,8 +1,12 @@ # Changelog ## 3.dev +* Fix: Using icalendar's tzinfo on created datetime to fix issue with icalendar +* Enhancement: Added free-busy report +* Enhancement: Added 'max_freebusy_occurrences` setting to avoid potential DOS on reports * Enhancement: remove unexpected control codes from uploaded items * Drop: remove unused requirement "typeguard" +* Improve: Refactored some date parsing code ## 3.2.2 * Enhancement: add support for auth.type=denyall (will be default for security reasons in upcoming releases) diff --git a/DOCUMENTATION.md b/DOCUMENTATION.md index c012b667..15f54d0b 100644 --- a/DOCUMENTATION.md +++ b/DOCUMENTATION.md @@ -1023,6 +1023,18 @@ RabbitMQ queue type for the topic. Default: classic +#### reporting +##### max_freebusy_occurrence + +When returning a free-busy report, a list of busy time occurrences are +generated based on a given time frame. Large time frames could +generate a lot of occurrences based on the time frame supplied. This +setting limits the lookup to prevent potential denial of service +attacks on large time frames. If the limit is reached, an HTTP error +is thrown instead of returning the results. + +Default: 10000 + ## Supported Clients Radicale has been tested with: diff --git a/config b/config index a9fe9da7..829ad7db 100644 --- a/config +++ b/config @@ -172,3 +172,9 @@ #rabbitmq_endpoint = #rabbitmq_topic = #rabbitmq_queue_type = classic + +[reporting] + +# When returning a free-busy report, limit the number of returned +# occurences per event to prevent DOS attacks. +#max_freebusy_occurrence = 10000 \ No newline at end of file diff --git a/radicale/app/report.py b/radicale/app/report.py index f0c50214..9d57b389 100644 --- a/radicale/app/report.py +++ b/radicale/app/report.py @@ -28,6 +28,7 @@ from typing import (Any, Callable, Iterable, Iterator, List, Optional, Sequence, Tuple, Union) from urllib.parse import unquote, urlparse +import vobject import vobject.base from vobject.base import ContentLine @@ -38,11 +39,110 @@ from radicale.item import filter as radicale_filter from radicale.log import logger +def free_busy_report(base_prefix: str, path: str, xml_request: Optional[ET.Element], + collection: storage.BaseCollection, encoding: str, + unlock_storage_fn: Callable[[], None], + max_occurrence: int + ) -> Tuple[int, Union[ET.Element, str]]: + # NOTE: this function returns both an Element and a string because + # free-busy reports are an edge-case on the return type according + # to the spec. + + multistatus = ET.Element(xmlutils.make_clark("D:multistatus")) + if xml_request is None: + return client.MULTI_STATUS, multistatus + root = xml_request + if (root.tag == xmlutils.make_clark("C:free-busy-query") and + collection.tag != "VCALENDAR"): + logger.warning("Invalid REPORT method %r on %r requested", + xmlutils.make_human_tag(root.tag), path) + return client.FORBIDDEN, xmlutils.webdav_error("D:supported-report") + + time_range_element = root.find(xmlutils.make_clark("C:time-range")) + assert isinstance(time_range_element, ET.Element) + + # Build a single filter from the free busy query for retrieval + # TODO: filter for VFREEBUSY in additional to VEVENT but + # test_filter doesn't support that yet. + vevent_cf_element = ET.Element(xmlutils.make_clark("C:comp-filter"), + attrib={'name': 'VEVENT'}) + vevent_cf_element.append(time_range_element) + vcalendar_cf_element = ET.Element(xmlutils.make_clark("C:comp-filter"), + attrib={'name': 'VCALENDAR'}) + vcalendar_cf_element.append(vevent_cf_element) + filter_element = ET.Element(xmlutils.make_clark("C:filter")) + filter_element.append(vcalendar_cf_element) + filters = (filter_element,) + + # First pull from storage + retrieved_items = list(collection.get_filtered(filters)) + # !!! Don't access storage after this !!! + unlock_storage_fn() + + cal = vobject.iCalendar() + collection_tag = collection.tag + while retrieved_items: + # Second filtering before evaluating occurrences. + # ``item.vobject_item`` might be accessed during filtering. + # Don't keep reference to ``item``, because VObject requires a lot of + # memory. + item, filter_matched = retrieved_items.pop(0) + if not filter_matched: + try: + if not test_filter(collection_tag, item, filter_element): + continue + except ValueError as e: + raise ValueError("Failed to free-busy filter item %r from %r: %s" % + (item.href, collection.path, e)) from e + except Exception as e: + raise RuntimeError("Failed to free-busy filter item %r from %r: %s" % + (item.href, collection.path, e)) from e + + fbtype = None + if item.component_name == 'VEVENT': + transp = getattr(item.vobject_item.vevent, 'transp', None) + if transp and transp.value != 'OPAQUE': + continue + + status = getattr(item.vobject_item.vevent, 'status', None) + if not status or status.value == 'CONFIRMED': + fbtype = 'BUSY' + elif status.value == 'CANCELLED': + fbtype = 'FREE' + elif status.value == 'TENTATIVE': + fbtype = 'BUSY-TENTATIVE' + else: + # Could do fbtype = status.value for x-name, I prefer this + fbtype = 'BUSY' + + # TODO: coalesce overlapping periods + + if max_occurrence > 0: + n_occurrences = max_occurrence+1 + else: + n_occurrences = 0 + occurrences = radicale_filter.time_range_fill(item.vobject_item, + time_range_element, + "VEVENT", + n=n_occurrences) + if len(occurrences) >= max_occurrence: + raise ValueError("FREEBUSY occurrences limit of {} hit" + .format(max_occurrence)) + + for occurrence in occurrences: + vfb = cal.add('vfreebusy') + vfb.add('dtstamp').value = item.vobject_item.vevent.dtstamp.value + vfb.add('dtstart').value, vfb.add('dtend').value = occurrence + if fbtype: + vfb.add('fbtype').value = fbtype + return (client.OK, cal.serialize()) + + def xml_report(base_prefix: str, path: str, xml_request: Optional[ET.Element], collection: storage.BaseCollection, encoding: str, unlock_storage_fn: Callable[[], None] ) -> Tuple[int, ET.Element]: - """Read and answer REPORT requests. + """Read and answer REPORT requests that return XML. Read rfc3253-3.6 for info. @@ -426,13 +526,28 @@ class ApplicationPartReport(ApplicationBase): else: assert item.collection is not None collection = item.collection - try: - status, xml_answer = xml_report( - base_prefix, path, xml_content, collection, self._encoding, - lock_stack.close) - except ValueError as e: - logger.warning( - "Bad REPORT request on %r: %s", path, e, exc_info=True) - return httputils.BAD_REQUEST - headers = {"Content-Type": "text/xml; charset=%s" % self._encoding} - return status, headers, self._xml_response(xml_answer) + + if xml_content is not None and \ + xml_content.tag == xmlutils.make_clark("C:free-busy-query"): + max_occurrence = self.configuration.get("reporting", "max_freebusy_occurrence") + try: + status, body = free_busy_report( + base_prefix, path, xml_content, collection, self._encoding, + lock_stack.close, max_occurrence) + except ValueError as e: + logger.warning( + "Bad REPORT request on %r: %s", path, e, exc_info=True) + return httputils.BAD_REQUEST + headers = {"Content-Type": "text/calendar; charset=%s" % self._encoding} + return status, headers, str(body) + else: + try: + status, xml_answer = xml_report( + base_prefix, path, xml_content, collection, self._encoding, + lock_stack.close) + except ValueError as e: + logger.warning( + "Bad REPORT request on %r: %s", path, e, exc_info=True) + return httputils.BAD_REQUEST + headers = {"Content-Type": "text/xml; charset=%s" % self._encoding} + return status, headers, self._xml_response(xml_answer) diff --git a/radicale/config.py b/radicale/config.py index d1b66251..0515813b 100644 --- a/radicale/config.py +++ b/radicale/config.py @@ -297,7 +297,13 @@ DEFAULT_CONFIG_SCHEMA: types.CONFIG_SCHEMA = OrderedDict([ "help": "mask passwords in logs", "type": bool})])), ("headers", OrderedDict([ - ("_allow_extra", str)]))]) + ("_allow_extra", str)])), + ("reporting", OrderedDict([ + ("max_freebusy_occurrence", { + "value": "10000", + "help": "number of occurrences per event when reporting", + "type": positive_int})])) + ]) def parse_compound_paths(*compound_paths: Optional[str] diff --git a/radicale/item/filter.py b/radicale/item/filter.py index 141845ce..cb3e8cdb 100644 --- a/radicale/item/filter.py +++ b/radicale/item/filter.py @@ -48,10 +48,34 @@ def date_to_datetime(d: date) -> datetime: if not isinstance(d, datetime): d = datetime.combine(d, datetime.min.time()) if not d.tzinfo: - d = d.replace(tzinfo=timezone.utc) + # NOTE: using vobject's UTC as it wasn't playing well with datetime's. + d = d.replace(tzinfo=vobject.icalendar.utc) return d +def parse_time_range(time_filter: ET.Element) -> Tuple[datetime, datetime]: + start_text = time_filter.get("start") + end_text = time_filter.get("end") + if start_text: + start = datetime.strptime( + start_text, "%Y%m%dT%H%M%SZ").replace( + tzinfo=timezone.utc) + else: + start = DATETIME_MIN + if end_text: + end = datetime.strptime( + end_text, "%Y%m%dT%H%M%SZ").replace( + tzinfo=timezone.utc) + else: + end = DATETIME_MAX + return start, end + + +def time_range_timestamps(time_filter: ET.Element) -> Tuple[int, int]: + start, end = parse_time_range(time_filter) + return (math.floor(start.timestamp()), math.ceil(end.timestamp())) + + def comp_match(item: "item.Item", filter_: ET.Element, level: int = 0) -> bool: """Check whether the ``item`` matches the comp ``filter_``. @@ -147,21 +171,10 @@ def time_range_match(vobject_item: vobject.base.Component, """Check whether the component/property ``child_name`` of ``vobject_item`` matches the time-range ``filter_``.""" - start_text = filter_.get("start") - end_text = filter_.get("end") - if not start_text and not end_text: + if not filter_.get("start") and not filter_.get("end"): return False - if start_text: - start = datetime.strptime(start_text, "%Y%m%dT%H%M%SZ") - else: - start = datetime.min - if end_text: - end = datetime.strptime(end_text, "%Y%m%dT%H%M%SZ") - else: - end = datetime.max - start = start.replace(tzinfo=timezone.utc) - end = end.replace(tzinfo=timezone.utc) + start, end = parse_time_range(filter_) matched = False def range_fn(range_start: datetime, range_end: datetime, @@ -181,6 +194,35 @@ def time_range_match(vobject_item: vobject.base.Component, return matched +def time_range_fill(vobject_item: vobject.base.Component, + filter_: ET.Element, child_name: str, n: int = 1 + ) -> List[Tuple[datetime, datetime]]: + """Create a list of ``n`` occurances from the component/property ``child_name`` + of ``vobject_item``.""" + if not filter_.get("start") and not filter_.get("end"): + return [] + + start, end = parse_time_range(filter_) + ranges: List[Tuple[datetime, datetime]] = [] + + def range_fn(range_start: datetime, range_end: datetime, + is_recurrence: bool) -> bool: + nonlocal ranges + if start < range_end and range_start < end: + ranges.append((range_start, range_end)) + if n > 0 and len(ranges) >= n: + return True + if end < range_start and not is_recurrence: + return True + return False + + def infinity_fn(range_start: datetime) -> bool: + return False + + visit_time_ranges(vobject_item, child_name, range_fn, infinity_fn) + return ranges + + def visit_time_ranges(vobject_item: vobject.base.Component, child_name: str, range_fn: Callable[[datetime, datetime, bool], bool], infinity_fn: Callable[[datetime], bool]) -> None: @@ -543,20 +585,7 @@ def simplify_prefilters(filters: Iterable[ET.Element], collection_tag: str if time_filter.tag != xmlutils.make_clark("C:time-range"): simple = False continue - start_text = time_filter.get("start") - end_text = time_filter.get("end") - if start_text: - start = math.floor(datetime.strptime( - start_text, "%Y%m%dT%H%M%SZ").replace( - tzinfo=timezone.utc).timestamp()) - else: - start = TIMESTAMP_MIN - if end_text: - end = math.ceil(datetime.strptime( - end_text, "%Y%m%dT%H%M%SZ").replace( - tzinfo=timezone.utc).timestamp()) - else: - end = TIMESTAMP_MAX + start, end = time_range_timestamps(time_filter) return tag, start, end, simple return tag, TIMESTAMP_MIN, TIMESTAMP_MAX, simple return None, TIMESTAMP_MIN, TIMESTAMP_MAX, simple diff --git a/radicale/tests/__init__.py b/radicale/tests/__init__.py index 528a98b2..ceb155b4 100644 --- a/radicale/tests/__init__.py +++ b/radicale/tests/__init__.py @@ -31,11 +31,12 @@ from io import BytesIO from typing import Any, Dict, List, Optional, Tuple, Union import defusedxml.ElementTree as DefusedET +import vobject import radicale from radicale import app, config, types, xmlutils -RESPONSES = Dict[str, Union[int, Dict[str, Tuple[int, ET.Element]]]] +RESPONSES = Dict[str, Union[int, Dict[str, Tuple[int, ET.Element]], vobject.base.Component]] # Enable debug output radicale.log.logger.setLevel(logging.DEBUG) @@ -107,8 +108,7 @@ class BaseTest: def parse_responses(text: str) -> RESPONSES: xml = DefusedET.fromstring(text) assert xml.tag == xmlutils.make_clark("D:multistatus") - path_responses: Dict[str, Union[ - int, Dict[str, Tuple[int, ET.Element]]]] = {} + path_responses: RESPONSES = {} for response in xml.findall(xmlutils.make_clark("D:response")): href = response.find(xmlutils.make_clark("D:href")) assert href.text not in path_responses @@ -133,6 +133,12 @@ class BaseTest: path_responses[href.text] = prop_responses return path_responses + @staticmethod + def parse_free_busy(text: str) -> RESPONSES: + path_responses: RESPONSES = {} + path_responses[""] = vobject.readOne(text) + return path_responses + def get(self, path: str, check: Optional[int] = 200, **kwargs ) -> Tuple[int, str]: assert "data" not in kwargs @@ -177,13 +183,18 @@ class BaseTest: return status, responses def report(self, path: str, data: str, check: Optional[int] = 207, + is_xml: Optional[bool] = True, **kwargs) -> Tuple[int, RESPONSES]: status, _, answer = self.request("REPORT", path, data, check=check, **kwargs) if status < 200 or 300 <= status: return status, {} assert answer is not None - return status, self.parse_responses(answer) + if is_xml: + parsed = self.parse_responses(answer) + else: + parsed = self.parse_free_busy(answer) + return status, parsed def delete(self, path: str, check: Optional[int] = 200, **kwargs ) -> Tuple[int, RESPONSES]: diff --git a/radicale/tests/static/event10.ics b/radicale/tests/static/event10.ics new file mode 100644 index 00000000..3faa034d --- /dev/null +++ b/radicale/tests/static/event10.ics @@ -0,0 +1,36 @@ +BEGIN:VCALENDAR +PRODID:-//Mozilla.org/NONSGML Mozilla Calendar V1.1//EN +VERSION:2.0 +BEGIN:VTIMEZONE +TZID:Europe/Paris +X-LIC-LOCATION:Europe/Paris +BEGIN:DAYLIGHT +TZOFFSETFROM:+0100 +TZOFFSETTO:+0200 +TZNAME:CEST +DTSTART:19700329T020000 +RRULE:FREQ=YEARLY;BYDAY=-1SU;BYMONTH=3 +END:DAYLIGHT +BEGIN:STANDARD +TZOFFSETFROM:+0200 +TZOFFSETTO:+0100 +TZNAME:CET +DTSTART:19701025T030000 +RRULE:FREQ=YEARLY;BYDAY=-1SU;BYMONTH=10 +END:STANDARD +END:VTIMEZONE +BEGIN:VEVENT +CREATED:20130902T150157Z +LAST-MODIFIED:20130902T150158Z +DTSTAMP:20130902T150158Z +UID:event10 +SUMMARY:Event +CATEGORIES:some_category1,another_category2 +ORGANIZER:mailto:unclesam@example.com +ATTENDEE;ROLE=REQ-PARTICIPANT;PARTSTAT=TENTATIVE;CN=Jane Doe:MAILTO:janedoe@example.com +ATTENDEE;ROLE=REQ-PARTICIPANT;DELEGATED-FROM="MAILTO:bob@host.com";PARTSTAT=ACCEPTED;CN=John Doe:MAILTO:johndoe@example.com +DTSTART;TZID=Europe/Paris:20130901T180000 +DTEND;TZID=Europe/Paris:20130901T190000 +STATUS:CANCELLED +END:VEVENT +END:VCALENDAR diff --git a/radicale/tests/test_base.py b/radicale/tests/test_base.py index dd2a2534..fc708ebc 100644 --- a/radicale/tests/test_base.py +++ b/radicale/tests/test_base.py @@ -25,6 +25,7 @@ import posixpath from typing import Any, Callable, ClassVar, Iterable, List, Optional, Tuple import defusedxml.ElementTree as DefusedET +import vobject from radicale import storage, xmlutils from radicale.tests import RESPONSES, BaseTest @@ -1360,10 +1361,45 @@ permissions: RrWw""") """) assert len(responses) == 1 response = responses[event_path] - assert not isinstance(response, int) + assert isinstance(response, dict) status, prop = response["D:getetag"] assert status == 200 and prop.text + def test_report_free_busy(self) -> None: + """Test free busy report on a few items""" + calendar_path = "/calendar.ics/" + self.mkcalendar(calendar_path) + for i in (1, 2, 10): + filename = "event{}.ics".format(i) + event = get_file_content(filename) + self.put(posixpath.join(calendar_path, filename), event) + code, responses = self.report(calendar_path, """\ + + + +""", 200, is_xml=False) + for response in responses.values(): + assert isinstance(response, vobject.base.Component) + assert len(responses) == 1 + vcalendar = list(responses.values())[0] + assert isinstance(vcalendar, vobject.base.Component) + assert len(vcalendar.vfreebusy_list) == 3 + types = {} + for vfb in vcalendar.vfreebusy_list: + fbtype_val = vfb.fbtype.value + if fbtype_val not in types: + types[fbtype_val] = 0 + types[fbtype_val] += 1 + assert types == {'BUSY': 2, 'FREE': 1} + + # Test max_freebusy_occurrence limit + self.configure({"reporting": {"max_freebusy_occurrence": 1}}) + code, responses = self.report(calendar_path, """\ + + + +""", 400, is_xml=False) + def _report_sync_token( self, calendar_path: str, sync_token: Optional[str] = None ) -> Tuple[str, RESPONSES]: