From 802037cf2239d9fbe0824e25602d45f845dd4ec6 Mon Sep 17 00:00:00 2001 From: Peter Bieringer Date: Sun, 20 Jul 2025 17:29:09 +0200 Subject: [PATCH 1/9] add a function to format unixtime to string --- radicale/utils.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/radicale/utils.py b/radicale/utils.py index 1b5a016c..5c8d4100 100644 --- a/radicale/utils.py +++ b/radicale/utils.py @@ -17,6 +17,7 @@ # You should have received a copy of the GNU General Public License # along with Radicale. If not, see . +import datetime import os import ssl import sys @@ -46,6 +47,10 @@ ADDRESS_TYPE = Union[Tuple[Union[str, bytes, bytearray], int], Tuple[str, int, int, int]] +# Max YEAR in datetime in unixtime +DATETIME_MAX_UNIXTIME: int = (datetime.MAXYEAR - 1970) * 365 * 24 * 60 * 60 + + def load_plugin(internal_types: Sequence[str], module_name: str, class_name: str, base_class: Type[_T_co], configuration: "config.Configuration") -> _T_co: @@ -244,3 +249,12 @@ def user_groups_as_string(): username = os.getlogin() s = "user=%s" % (username) return s + + +def format_ut(unixtime: int) -> str: + if unixtime < DATETIME_MAX_UNIXTIME: + dt = datetime.datetime.fromtimestamp(unixtime, datetime.UTC) + r = str(unixtime) + "(" + dt.strftime('%Y-%m-%dT%H:%M:%SZ') + ")" + else: + r = str(unixtime) + "(>MAX:" + str(DATETIME_MAX_UNIXTIME) + ")" + return r From c8e42c7edc5cc2ad5c348ea8b818a305402b9a3c Mon Sep 17 00:00:00 2001 From: Peter Bieringer Date: Sun, 20 Jul 2025 17:30:03 +0200 Subject: [PATCH 2/9] add test cases for filter without comp-filter --- radicale/tests/test_base.py | 84 +++++++++++++++++++++++++++++++++++++ 1 file changed, 84 insertions(+) diff --git a/radicale/tests/test_base.py b/radicale/tests/test_base.py index d2b26949..eb25bd1f 100644 --- a/radicale/tests/test_base.py +++ b/radicale/tests/test_base.py @@ -1214,6 +1214,90 @@ permissions: RrWw""") """], items=(9,)) assert "/calendar.ics/event9.ics" not in answer + def test_time_range_filter_without_comp_filter(self) -> None: + """Report request with time-range filter without comp-filter on events.""" + answer = self._test_filter(["""\ + + +"""], "event", items=range(1, 6)) + assert "/calendar.ics/event1.ics" in answer + assert "/calendar.ics/event2.ics" in answer + assert "/calendar.ics/event3.ics" in answer + assert "/calendar.ics/event4.ics" in answer + assert "/calendar.ics/event5.ics" in answer + answer = self._test_filter(["""\ + + +"""], items=range(1, 6)) + assert "/calendar.ics/event1.ics" not in answer + assert "/calendar.ics/event2.ics" in answer + assert "/calendar.ics/event3.ics" in answer + assert "/calendar.ics/event4.ics" in answer + assert "/calendar.ics/event5.ics" in answer + answer = self._test_filter(["""\ + + +"""], items=range(1, 6)) + assert "/calendar.ics/event1.ics" not in answer + assert "/calendar.ics/event2.ics" not in answer + assert "/calendar.ics/event3.ics" in answer + assert "/calendar.ics/event4.ics" in answer + assert "/calendar.ics/event5.ics" in answer + answer = self._test_filter(["""\ + + +"""], items=range(1, 6)) + assert "/calendar.ics/event1.ics" not in answer + assert "/calendar.ics/event2.ics" not in answer + assert "/calendar.ics/event3.ics" in answer + assert "/calendar.ics/event4.ics" not in answer + assert "/calendar.ics/event5.ics" not in answer + answer = self._test_filter(["""\ + + +"""], items=range(1, 6)) + assert "/calendar.ics/event1.ics" not in answer + assert "/calendar.ics/event2.ics" not in answer + assert "/calendar.ics/event3.ics" not in answer + assert "/calendar.ics/event4.ics" not in answer + assert "/calendar.ics/event5.ics" not in answer + # HACK: VObject doesn't match RECURRENCE-ID to recurrences, the + # overwritten recurrence is still used for filtering. + answer = self._test_filter(["""\ + + +"""], items=(6, 7, 8, 9)) + assert "/calendar.ics/event6.ics" in answer + assert "/calendar.ics/event7.ics" in answer + assert "/calendar.ics/event8.ics" in answer + assert "/calendar.ics/event9.ics" in answer + answer = self._test_filter(["""\ + + +"""], items=(6, 7, 8, 9)) + assert "/calendar.ics/event6.ics" in answer + assert "/calendar.ics/event7.ics" in answer + assert "/calendar.ics/event8.ics" in answer + assert "/calendar.ics/event9.ics" not in answer + answer = self._test_filter(["""\ + + +"""], items=(6, 7, 8, 9)) + assert "/calendar.ics/event6.ics" not in answer + assert "/calendar.ics/event7.ics" not in answer + assert "/calendar.ics/event8.ics" not in answer + assert "/calendar.ics/event9.ics" not in answer + answer = self._test_filter(["""\ + + +"""], items=(9,)) + assert "/calendar.ics/event9.ics" in answer + answer = self._test_filter(["""\ + + +"""], items=(9,)) + assert "/calendar.ics/event9.ics" not in answer + def test_time_range_filter_events_rrule(self) -> None: """Report request with time-range filter on events with rrules.""" answer = self._test_filter(["""\ From 2209edfb88eac6f50a6090d20ba8c05114ea0bc4 Mon Sep 17 00:00:00 2001 From: Peter Bieringer Date: Sun, 20 Jul 2025 17:33:23 +0200 Subject: [PATCH 3/9] add some trace log --- radicale/storage/__init__.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/radicale/storage/__init__.py b/radicale/storage/__init__.py index da89bcf3..b9a6864e 100644 --- a/radicale/storage/__init__.py +++ b/radicale/storage/__init__.py @@ -2,7 +2,7 @@ # Copyright © 2014 Jean-Marc Martins # Copyright © 2012-2017 Guillaume Ayoub # Copyright © 2017-2022 Unrud -# Copyright © 2024-2024 Peter Bieringer +# Copyright © 2024-2025 Peter Bieringer # # This library is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -37,6 +37,7 @@ from radicale import item as radicale_item from radicale import types, utils from radicale.item import filter as radicale_filter from radicale.log import logger +from radicale.utils import format_ut INTERNAL_TYPES: Sequence[str] = ("multifilesystem", "multifilesystem_nolock",) @@ -153,12 +154,17 @@ class BaseCollection: return tag, start, end, simple = radicale_filter.simplify_prefilters( filters, self.tag) + logger.debug("TRACE/STORAGE/get_filtered: prefilter tag=%s start=%s end=%s simple=%s", tag, format_ut(start), format_ut(end), simple) for item in self.get_all(): + logger.debug("TRACE/STORAGE/get_filtered: component_name=%s tag=%s", item.component_name, tag) if tag is not None and tag != item.component_name: continue istart, iend = item.time_range + logger.debug("TRACE/STORAGE/get_filtered: istart=%s iend=%s", format_ut(istart), format_ut(iend)) if istart >= end or iend <= start: + logger.debug("TRACE/STORAGE/get_filtered: skip iuid=%s", item.uid) continue + logger.debug("TRACE/STORAGE/get_filtered: add iuid=%s", item.uid) yield item, simple and (start <= istart or iend <= end) def has_uid(self, uid: str) -> bool: From b756e21c31197b30d5be5df54afd7b5293e4cbbf Mon Sep 17 00:00:00 2001 From: Peter Bieringer Date: Sun, 20 Jul 2025 17:38:31 +0200 Subject: [PATCH 4/9] add some trace logging --- radicale/item/filter.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/radicale/item/filter.py b/radicale/item/filter.py index 0e62be8f..6f34df69 100644 --- a/radicale/item/filter.py +++ b/radicale/item/filter.py @@ -32,6 +32,7 @@ import vobject from radicale import item, xmlutils from radicale.log import logger +from radicale.utils import format_ut DAY: timedelta = timedelta(days=1) SECOND: timedelta = timedelta(seconds=1) @@ -98,6 +99,7 @@ def comp_match(item: "item.Item", filter_: ET.Element, level: int = 0) -> bool: # HACK: the filters are tested separately against all components name = filter_.get("name", "").upper() + logger.debug("TRACE/ITEM/FILTER/comp_match: name=%s level=%d", name, level) if level == 0: tag = item.name @@ -233,6 +235,7 @@ def time_range_match(vobject_item: vobject.base.Component, def infinity_fn(start: datetime) -> bool: return False + logger.debug("TRACE/ITEM/FILTER/time_range_match: start=(%s) end=(%s) child_name=%s", start, end, child_name) visit_time_ranges(vobject_item, child_name, range_fn, infinity_fn) return matched @@ -289,6 +292,8 @@ def visit_time_ranges(vobject_item: vobject.base.Component, child_name: str, # recurrences too. This is not respected and client don't seem to bother # either. + logger.debug("TRACE/ITEM/FILTER/visit_time_ranges: child_name=%s", child_name) + def getrruleset(child: vobject.base.Component, ignore: Sequence[date] ) -> Tuple[Iterable[date], bool]: infinite = False @@ -516,6 +521,7 @@ def visit_time_ranges(vobject_item: vobject.base.Component, child_name: str, else: # Match a property + logger.debug("TRACE/ITEM/FILTER/get_children: child_name=%s property match", child_name) child = getattr(vobject_item, child_name.lower()) if isinstance(child.value, date): child_is_datetime = isinstance(child.value, datetime) @@ -605,6 +611,7 @@ def simplify_prefilters(filters: Iterable[ET.Element], collection_tag: str """ flat_filters = list(chain.from_iterable(filters)) simple = len(flat_filters) <= 1 + logger.debug("TRACE/ITEM/FILTER/simplify_prefilters: collection_tag=%s", collection_tag) for col_filter in flat_filters: if collection_tag != "VCALENDAR": simple = False @@ -632,6 +639,7 @@ def simplify_prefilters(filters: Iterable[ET.Element], collection_tag: str simple = False continue start, end = time_range_timestamps(time_filter) + logger.debug("TRACE/ITEM/FILTER/simplify_prefilters: found time-filter on level 1 tag=%s start=%d end=%d simple=%s", tag, start, end, simple) return tag, start, end, simple return tag, TIMESTAMP_MIN, TIMESTAMP_MAX, simple return None, TIMESTAMP_MIN, TIMESTAMP_MAX, simple From 18a61f209dcc7225c09246aa06ff911a866c0693 Mon Sep 17 00:00:00 2001 From: Peter Bieringer Date: Sun, 20 Jul 2025 17:42:43 +0200 Subject: [PATCH 5/9] catch time-filter on level 0 --- radicale/item/filter.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/radicale/item/filter.py b/radicale/item/filter.py index 6f34df69..4cddf0d8 100644 --- a/radicale/item/filter.py +++ b/radicale/item/filter.py @@ -622,7 +622,14 @@ def simplify_prefilters(filters: Iterable[ET.Element], collection_tag: str continue simple &= len(col_filter) <= 1 for comp_filter in col_filter: + logger.debug("TRACE/ITEM/FILTER/simplify_prefilters: filter.tag=%s simple=%s", comp_filter.tag, simple) + if comp_filter.tag == xmlutils.make_clark("C:time-range") and simple is True: + # time-filter found on level 0 + start, end = time_range_timestamps(comp_filter) + logger.debug("TRACE/ITEM/FILTER/simplify_prefilters: found time-filter on level 0 start=%r(%d) end=%r(%d) simple=%s", format_ut(start), start, format_ut(end), end, simple) + return None, start, end, simple if comp_filter.tag != xmlutils.make_clark("C:comp-filter"): + logger.debug("TRACE/ITEM/FILTER/simplify_prefilters: no comp-filter on level 0") simple = False continue tag = comp_filter.get("name", "").upper() From 41ac453eb7f08817ee059c6ec2360e002ff27946 Mon Sep 17 00:00:00 2001 From: Peter Bieringer Date: Sun, 20 Jul 2025 17:47:18 +0200 Subject: [PATCH 6/9] try applying timerange-filter on VCALENDAR entries --- radicale/item/filter.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/radicale/item/filter.py b/radicale/item/filter.py index 4cddf0d8..b846023a 100644 --- a/radicale/item/filter.py +++ b/radicale/item/filter.py @@ -144,13 +144,24 @@ def comp_match(item: "item.Item", filter_: ET.Element, level: int = 0) -> bool: trigger = subcomp.trigger.value for child in filter_: if child.tag == xmlutils.make_clark("C:prop-filter"): + logger.debug("TRACE/ITEM/FILTER/comp_match: prop-filter level=%d", level) if not any(prop_match(comp, child, "C") for comp in components): return False elif child.tag == xmlutils.make_clark("C:time-range"): + logger.debug("TRACE/ITEM/FILTER/comp_match: time-range level=%d tag=%s", level, tag) + if (level == 0) and (name == "VCALENDAR"): + for name_try in ("VTODO", "VEVENT", "VJOURNAL"): + try: + if time_range_match(item.vobject_item, filter_[0], name_try, trigger): + return True + except Exception: + continue + return False if not time_range_match(item.vobject_item, filter_[0], tag, trigger): return False elif child.tag == xmlutils.make_clark("C:comp-filter"): + logger.debug("TRACE/ITEM/FILTER/comp_match: comp-filter level=%d", level) if not comp_match(item, child, level=level + 1): return False else: From 0a6cc90e5f1e81c29110a8b7ab53a85f713ced2f Mon Sep 17 00:00:00 2001 From: Peter Bieringer Date: Sun, 20 Jul 2025 17:50:34 +0200 Subject: [PATCH 7/9] extend changelog for fix (rebase) --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index a91f5630..59e13e28 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,7 @@ * Fix: report with enabled expand honors now provided filter proper * Improve: add options [logging] trace_on_debug and trace_filter for supporting trace logging * Fix: catch case where getpwuid is not returning a username +* Fix: add support for query without comp-type ## 3.5.4 * Improve: item filter enhanced for 3rd level supporting VALARM and honoring TRIGGER (offset or absolute) From 48ae4d1d6ecba5843489378c811c5420ab984001 Mon Sep 17 00:00:00 2001 From: Peter Bieringer Date: Sun, 20 Jul 2025 17:55:28 +0200 Subject: [PATCH 8/9] python < 3.11 support --- radicale/utils.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/radicale/utils.py b/radicale/utils.py index 5c8d4100..857a58f5 100644 --- a/radicale/utils.py +++ b/radicale/utils.py @@ -253,7 +253,10 @@ def user_groups_as_string(): def format_ut(unixtime: int) -> str: if unixtime < DATETIME_MAX_UNIXTIME: - dt = datetime.datetime.fromtimestamp(unixtime, datetime.UTC) + if sys.version_info < (3, 11): + dt = datetime.datetime.utcfromtimestamp(unixtime) + else: + dt = datetime.datetime.fromtimestamp(unixtime, datetime.UTC) r = str(unixtime) + "(" + dt.strftime('%Y-%m-%dT%H:%M:%SZ') + ")" else: r = str(unixtime) + "(>MAX:" + str(DATETIME_MAX_UNIXTIME) + ")" From c84d09b4dc97f0e86149edf68e67e70465b81770 Mon Sep 17 00:00:00 2001 From: Peter Bieringer Date: Sun, 20 Jul 2025 18:00:39 +0200 Subject: [PATCH 9/9] workaround for windows --- radicale/utils.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/radicale/utils.py b/radicale/utils.py index 857a58f5..096864b6 100644 --- a/radicale/utils.py +++ b/radicale/utils.py @@ -252,6 +252,9 @@ def user_groups_as_string(): def format_ut(unixtime: int) -> str: + if sys.platform == "win32": + # TODO check how to support this better + return str(unixtime) if unixtime < DATETIME_MAX_UNIXTIME: if sys.version_info < (3, 11): dt = datetime.datetime.utcfromtimestamp(unixtime)