From 9d591bd5144c97ae3803512b6c22cd5ce1dfd0f9 Mon Sep 17 00:00:00 2001 From: Georgiy Date: Tue, 1 Jul 2025 23:00:25 +0300 Subject: [PATCH 01/35] (#1812) Work on expand events time-range filter processing --- radicale/app/report.py | 81 ++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 78 insertions(+), 3 deletions(-) diff --git a/radicale/app/report.py b/radicale/app/report.py index c6966777..8bda606b 100644 --- a/radicale/app/report.py +++ b/radicale/app/report.py @@ -224,9 +224,17 @@ def xml_report(base_prefix: str, path: str, xml_request: Optional[ET.Element], root.findall(xmlutils.make_clark("C:filter")) + root.findall(xmlutils.make_clark("CR:filter"))) + # extract time-range filter for processing after main filters + time_range_element = None + non_time_range_filters = [] + for filter_ in filters: + time_range_element = filter_.find(".//" + xmlutils.make_clark("C:time-range")) + if time_range_element is None: + non_time_range_filters.append(filter_) + # Retrieve everything required for finishing the request. retrieved_items = list(retrieve_items( - base_prefix, path, collection, hreferences, filters, multistatus)) + base_prefix, path, collection, hreferences, non_time_range_filters, multistatus)) collection_tag = collection.tag # !!! Don't access storage after this !!! unlock_storage_fn() @@ -239,7 +247,7 @@ def xml_report(base_prefix: str, path: str, xml_request: Optional[ET.Element], if filters and not filters_matched: try: if not all(test_filter(collection_tag, item, filter_) - for filter_ in filters): + for filter_ in non_time_range_filters): continue except ValueError as e: raise ValueError("Failed to filter item %r from %r: %s" % @@ -248,6 +256,13 @@ def xml_report(base_prefix: str, path: str, xml_request: Optional[ET.Element], raise RuntimeError("Failed to filter item %r from %r: %s" % (item.href, collection.path, e)) from e + # Filtering non-recurring events by time-range + if (time_range_element is not None) and not hasattr(item, 'rrule'): + start, end = radicale_filter.time_range_timestamps(time_range_element) + istart, iend = item.time_range + if istart >= end or iend <= start: + continue + found_props = [] not_found_props = [] @@ -280,8 +295,18 @@ def xml_report(base_prefix: str, path: str, xml_request: Optional[ET.Element], end, '%Y%m%dT%H%M%SZ' ).replace(tzinfo=datetime.timezone.utc) + time_range_start = None + time_range_end = None + + if time_range_element is not None: + time_range_start, time_range_end = radicale_filter.parse_time_range(time_range_element) + expanded_element = _expand( - element, copy.copy(item), start, end) + element=element, item=copy.copy(item), + start=start, end=end, + time_range_start=time_range_start, time_range_end=time_range_end, + ) + found_props.append(expanded_element) else: found_props.append(element) @@ -303,6 +328,8 @@ def _expand( item: radicale_item.Item, start: datetime.datetime, end: datetime.datetime, + time_range_start: Optional[datetime.datetime] = None, + time_range_end: Optional[datetime.datetime] = None, ) -> ET.Element: vevent_component: vobject.base.Component = copy.copy(item.vobject_item) @@ -347,6 +374,13 @@ def _expand( for recurrence_dt in recurrences: recurrence_utc = recurrence_dt.astimezone(datetime.timezone.utc) + + if time_range_start is not None and time_range_end is not None: + dtstart = recurrence_dt if all_day_event else recurrence_utc + dtend = dtstart + duration if duration else dtstart + if not (dtstart < time_range_end and dtend > time_range_start): + continue + i_overridden, vevent = _find_overridden(i_overridden, vevents_overridden, recurrence_utc, dt_format) if not vevent: @@ -377,6 +411,47 @@ def _expand( else: vevent_component.add(vevent) + # Filter overridden events and vevent_recurrence if recurrences is empty + # Todo: optimize that code + if time_range_start is not None and time_range_end is not None: + filtered_vevents = [] + for vevent in vevents_overridden: + dtstart = vevent.dtstart.value + dtend = vevent.dtend.value if hasattr(vevent, 'dtend') else dtstart + + dtstart = datetime.datetime.strptime( + dtstart, "%Y%m%dT%H%M%SZ").replace( + tzinfo=datetime.timezone.utc) + dtend = datetime.datetime.strptime( + dtend, "%Y%m%dT%H%M%SZ").replace( + tzinfo=datetime.timezone.utc) + + if dtstart < time_range_end and dtend > time_range_start: + filtered_vevents.append(vevent) + + dtstart = vevent_recurrence.dtstart.value + dtend = vevent_recurrence.dtend.value if hasattr(vevent_recurrence, 'dtend') else dtstart + dtstart = datetime.datetime.strptime( + dtstart, "%Y%m%dT%H%M%SZ").replace( + tzinfo=datetime.timezone.utc) + dtend = datetime.datetime.strptime( + dtend, "%Y%m%dT%H%M%SZ").replace( + tzinfo=datetime.timezone.utc) + + if filtered_vevents or (dtstart < time_range_end and dtend > time_range_start): + if filtered_vevents: + vevent_component.vevent = filtered_vevents[0] + for vevent in filtered_vevents[1:]: + vevent_component.add(vevent) + if dtstart < time_range_end and dtend > time_range_start: + if not filtered_vevents: + vevent_component.vevent = vevent_recurrence + else: + vevent_component.add(vevent_recurrence) + else: + element.text = "" + return element + element.text = vevent_component.serialize() return element From db646d4b24b17cc77818aae84227efa3eecbf213 Mon Sep 17 00:00:00 2001 From: David Greaves Date: Tue, 1 Jul 2025 20:48:27 +0100 Subject: [PATCH 02/35] add tests for issue #1812 A report should include: * An expanded event with an expand start inside a recurring event * An expanded event with an expand start and end inside a recurring event Signed-off-by: David Greaves --- radicale/tests/test_expand.py | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/radicale/tests/test_expand.py b/radicale/tests/test_expand.py index 1070bc77..77de697c 100644 --- a/radicale/tests/test_expand.py +++ b/radicale/tests/test_expand.py @@ -181,6 +181,32 @@ permissions: RrWw""") 1 ) + def test_report_with_expand_property_start_inside(self) -> None: + """Test report with expand property start inside""" + self._test_expand( + "event_daily_rrule", + "20060103T171500Z", + "20060105T000000Z", + ["RECURRENCE-ID:20060103T170000Z", "RECURRENCE-ID:20060104T170000Z"], + ["DTSTART:20060103T170000Z", "DTSTART:20060104T170000Z"], + [], + CONTAINS_TIMES, + 1 + ) + + def test_report_with_expand_property_just_inside(self) -> None: + """Test report with expand property start and end inside event""" + self._test_expand( + "event_daily_rrule", + "20060103T171500Z", + "20060103T171501Z", + ["RECURRENCE-ID:20060103T170000Z"], + ["DTSTART:20060103T170000Z"], + [], + CONTAINS_TIMES, + 1 + ) + def test_report_with_expand_property_all_day_event(self) -> None: """Test report with expand property for all day events""" self._test_expand( From 01bcc7d009a1d937395b69cc8bfff0ee7e652f7f Mon Sep 17 00:00:00 2001 From: David Greaves Date: Tue, 1 Jul 2025 21:19:26 +0100 Subject: [PATCH 03/35] add test for specific ics and query in issue#1812 Signed-off-by: David Greaves --- radicale/tests/static/event_issue1812.ics | 129 ++++++++++++++++++++++ radicale/tests/test_expand.py | 13 +++ 2 files changed, 142 insertions(+) create mode 100644 radicale/tests/static/event_issue1812.ics diff --git a/radicale/tests/static/event_issue1812.ics b/radicale/tests/static/event_issue1812.ics new file mode 100644 index 00000000..4f270fe7 --- /dev/null +++ b/radicale/tests/static/event_issue1812.ics @@ -0,0 +1,129 @@ +BEGIN:VCALENDAR +VERSION:2.0 +PRODID:DAVx5/4.4.6-beta.1-ose ical4j/3.2.19 +BEGIN:VTIMEZONE +TZID:Europe/London +BEGIN:STANDARD +DTSTART:19961027T020000 +RRULE:FREQ=YEARLY;BYMONTH=10;BYDAY=-1SU +TZNAME:GMT +TZOFFSETFROM:+0100 +TZOFFSETTO:+0000 +END:STANDARD +BEGIN:DAYLIGHT +DTSTART:19810329T010000 +RRULE:FREQ=YEARLY;BYMONTH=3;BYDAY=-1SU +TZNAME:BST +TZOFFSETFROM:+0000 +TZOFFSETTO:+0100 +END:DAYLIGHT +END:VTIMEZONE +BEGIN:VEVENT +UID:event_issue1812 +DTSTART;TZID=Europe/London:20230101T180000 +DTEND;TZID=Europe/London:20230101T233000 +CREATED:20230130T181142Z +DTSTAMP:20250515T182647Z +EXDATE;TZID=Europe/London:20231222T180000,20240112T180000,20240126T180000,2 + 0240329T180000,20241018T180000,20241129T180000,20241206T180000,20241213T18 + 0000 +EXDATE;TZID=Europe/London:20250521T180000 +EXDATE;TZID=Europe/London:20250515T180000 +RELATED-TO;RELTYPE=X-CALENDARSERVER-RECURRENCE-SET:3EF0E463-40EB-47FF-B825- + D474CE894708 +RRULE:FREQ=DAILY +SEQUENCE:11 +SUMMARY:TV Room +X-MOZ-GENERATION:23 +END:VEVENT +BEGIN:VEVENT +UID:event_issue1812 +RECURRENCE-ID;TZID=Europe/London:20240113T180000 +DTSTART;TZID=Europe/London:20240113T183000 +DTEND;TZID=Europe/London:20240113T230000 +DTSTAMP:20250515T182647Z +SEQUENCE:5 +SUMMARY:TV Room +END:VEVENT +BEGIN:VEVENT +UID:event_issue1812 +RECURRENCE-ID;TZID=Europe/London:20231227T180000 +DTSTART;TZID=Europe/London:20231227T203000 +DTEND;TZID=Europe/London:20231227T233000 +DTSTAMP:20250515T182647Z +SEQUENCE:3 +SUMMARY:TV Room +END:VEVENT +BEGIN:VEVENT +UID:event_issue1812 +RECURRENCE-ID;TZID=Europe/London:20231126T180000 +DTSTART;TZID=Europe/London:20231126T180000 +DTEND;TZID=Europe/London:20231126T223000 +DTSTAMP:20250515T182647Z +SEQUENCE:3 +SUMMARY:TV Room +END:VEVENT +BEGIN:VEVENT +UID:event_issue1812 +RECURRENCE-ID;TZID=Europe/London:20231225T180000 +DTSTART;TZID=Europe/London:20231225T211500 +DTEND;TZID=Europe/London:20231225T233000 +DTSTAMP:20250515T182647Z +SEQUENCE:3 +SUMMARY:TV Room +END:VEVENT +BEGIN:VEVENT +UID:event_issue1812 +RECURRENCE-ID;TZID=Europe/London:20231129T180000 +DTSTART;TZID=Europe/London:20231129T173000 +DTEND;TZID=Europe/London:20231129T233000 +DTSTAMP:20250515T182647Z +SEQUENCE:2 +SUMMARY:TV Room +END:VEVENT +BEGIN:VEVENT +UID:event_issue1812 +RECURRENCE-ID;TZID=Europe/London:20240220T180000 +DTSTART;TZID=Europe/London:20240220T173000 +DTEND;TZID=Europe/London:20240220T233000 +DTSTAMP:20250515T182647Z +SEQUENCE:5 +SUMMARY:TV Room +END:VEVENT +BEGIN:VEVENT +UID:event_issue1812 +RECURRENCE-ID;TZID=Europe/London:20240310T180000 +DTSTART;TZID=Europe/London:20240310T174500 +DTEND;TZID=Europe/London:20240310T233000 +DTSTAMP:20250515T182647Z +SEQUENCE:5 +SUMMARY:TV Room +END:VEVENT +BEGIN:VEVENT +UID:event_issue1812 +RECURRENCE-ID;TZID=Europe/London:20240324T180000 +DTSTART;TZID=Europe/London:20240324T183000 +DTEND;TZID=Europe/London:20240324T233000 +DTSTAMP:20250515T182648Z +SEQUENCE:6 +SUMMARY:TV Room +END:VEVENT +BEGIN:VEVENT +UID:event_issue1812 +RECURRENCE-ID;TZID=Europe/London:20241027T180000 +DTSTART;TZID=Europe/London:20241027T173000 +DTEND;TZID=Europe/London:20241027T233000 +DTSTAMP:20250515T182648Z +SEQUENCE:7 +SUMMARY:TV Room +END:VEVENT +BEGIN:VEVENT +UID:event_issue1812 +RECURRENCE-ID;TZID=Europe/London:20241226T180000 +DTSTART;TZID=Europe/London:20241226T193000 +DTEND;TZID=Europe/London:20241227T003000 +DTSTAMP:20250515T182648Z +SEQUENCE:10 +SUMMARY:TV Room +END:VEVENT +END:VCALENDAR diff --git a/radicale/tests/test_expand.py b/radicale/tests/test_expand.py index 77de697c..3055eafb 100644 --- a/radicale/tests/test_expand.py +++ b/radicale/tests/test_expand.py @@ -207,6 +207,19 @@ permissions: RrWw""") 1 ) + def test_report_with_expand_property_issue1812(self) -> None: + """Test report with expand property for issue 1812""" + self._test_expand( + "event_issue1812", + "20250627T183000Z", + "20250627T183001Z", + ["RECURRENCE-ID:20250627T180000Z"], + ["DTSTART:20250627T180000Z"], + [], + CONTAINS_TIMES, + 1 + ) + def test_report_with_expand_property_all_day_event(self) -> None: """Test report with expand property for all day events""" self._test_expand( From 097360139a865140844b6a3f7eeebba76e5488f6 Mon Sep 17 00:00:00 2001 From: Georgiy Date: Wed, 2 Jul 2025 23:30:17 +0300 Subject: [PATCH 04/35] (#1812) Refactored _expand function to process overridden VEVENTs and support EXDATE for recurrence filtering. Added temporary workaround to return base vevent_recurrence when filtered_vevents is empty to avoid 'list index out of range' error --- radicale/app/report.py | 110 +++++++++++++++++++++++++---------------- 1 file changed, 68 insertions(+), 42 deletions(-) diff --git a/radicale/app/report.py b/radicale/app/report.py index 8bda606b..1df045d0 100644 --- a/radicale/app/report.py +++ b/radicale/app/report.py @@ -332,6 +332,7 @@ def _expand( time_range_end: Optional[datetime.datetime] = None, ) -> ET.Element: vevent_component: vobject.base.Component = copy.copy(item.vobject_item) + logger.info("Expanding event %s", item.href) # Split the vevents included in the component into one that contains the # recurrence information and others that contain a recurrence id to @@ -350,6 +351,9 @@ def _expand( # rruleset.between computes with datetimes without timezone information start = start.replace(tzinfo=None) end = end.replace(tzinfo=None) + if time_range_start is not None and time_range_end is not None: + time_range_start = time_range_start.replace(tzinfo=None) + time_range_end = time_range_end.replace(tzinfo=None) for vevent in vevents_overridden: _strip_single_event(vevent, dt_format) @@ -358,10 +362,21 @@ def _expand( if hasattr(vevent_recurrence, "dtend"): duration = vevent_recurrence.dtend.value - vevent_recurrence.dtstart.value + # Handle EXDATE to limit expansion range + if hasattr(vevent_recurrence, 'exdate'): + exdates = vevent_recurrence.exdate.value + if not isinstance(exdates, list): + exdates = [exdates] # Convert single date to list + logger.debug("EXDATE values: %s", exdates) + latest_exdate = max(exdates) if exdates else None + if latest_exdate and end > latest_exdate: + end = min(end, latest_exdate) + rruleset = None if hasattr(vevent_recurrence, 'rrule'): rruleset = vevent_recurrence.getrruleset() + filtered_vevents = [] if rruleset: # This function uses datetimes internally without timezone info for dates recurrences = rruleset.between(start, end, inc=True) @@ -369,27 +384,32 @@ def _expand( _strip_component(vevent_component) _strip_single_event(vevent_recurrence, dt_format) - is_component_filled: bool = False i_overridden = 0 for recurrence_dt in recurrences: - recurrence_utc = recurrence_dt.astimezone(datetime.timezone.utc) + recurrence_utc = recurrence_dt if all_day_event else recurrence_dt.astimezone(datetime.timezone.utc) + logger.debug("Processing recurrence: %s (all_day_event: %s)", recurrence_utc, all_day_event) + # Apply time-range filter if time_range_start is not None and time_range_end is not None: - dtstart = recurrence_dt if all_day_event else recurrence_utc + dtstart = recurrence_utc dtend = dtstart + duration if duration else dtstart if not (dtstart < time_range_end and dtend > time_range_start): + logger.debug("Recurrence %s filtered out by time-range", recurrence_utc) continue + # Check for overridden instances i_overridden, vevent = _find_overridden(i_overridden, vevents_overridden, recurrence_utc, dt_format) if not vevent: - # We did not find an overridden instance, so create a new one + # Create new instance from recurrence vevent = copy.deepcopy(vevent_recurrence) # For all day events, the system timezone may influence the # results, so use recurrence_dt recurrence_id = recurrence_dt if all_day_event else recurrence_utc + logger.debug("Creating new VEVENT with RECURRENCE-ID: %s", recurrence_id) + vevent.recurrence_id = ContentLine( name='RECURRENCE-ID', value=recurrence_id, params={} @@ -405,54 +425,60 @@ def _expand( value=(recurrence_id + duration).strftime(dt_format), params={} ) - if not is_component_filled: - vevent_component.vevent = vevent - is_component_filled = True - else: - vevent_component.add(vevent) + filtered_vevents.append(vevent) - # Filter overridden events and vevent_recurrence if recurrences is empty - # Todo: optimize that code + # Filter overridden and recurrence base events if time_range_start is not None and time_range_end is not None: - filtered_vevents = [] - for vevent in vevents_overridden: + for vevent in vevents_overridden + [vevent_recurrence]: dtstart = vevent.dtstart.value dtend = vevent.dtend.value if hasattr(vevent, 'dtend') else dtstart + logger.debug( + "Filtering VEVENT with DTSTART: %s (type: %s), DTEND: %s (type: %s)", + dtstart, type(dtstart), dtend, type(dtend)) - dtstart = datetime.datetime.strptime( - dtstart, "%Y%m%dT%H%M%SZ").replace( - tzinfo=datetime.timezone.utc) - dtend = datetime.datetime.strptime( - dtend, "%Y%m%dT%H%M%SZ").replace( - tzinfo=datetime.timezone.utc) + # Handle string values for DTSTART/DTEND + if isinstance(dtstart, str): + try: + dtstart = datetime.datetime.strptime(dtstart, dt_format) + if all_day_event: + dtstart = dtstart.date() + except ValueError as e: + logger.warning("Invalid DTSTART format: %s, error: %s", dtstart, e) + continue + if isinstance(dtend, str): + try: + dtend = datetime.datetime.strptime(dtend, dt_format) + if all_day_event: + dtend = dtend.date() + except ValueError as e: + logger.warning("Invalid DTSTART format: %s, error: %s", dtstart, e) + continue + + # Convert to datetime for comparison + if all_day_event and isinstance(dtstart, datetime.date) and not isinstance(dtstart, datetime.datetime): + dtstart = datetime.datetime.fromordinal(dtstart.toordinal()).replace(tzinfo=None) + dtend = datetime.datetime.fromordinal(dtend.toordinal()).replace(tzinfo=None) + elif not all_day_event and isinstance(dtstart, datetime.datetime) \ + and isinstance(dtend, datetime.datetime): + dtstart = dtstart.replace(tzinfo=datetime.timezone.utc) + dtend = dtend.replace(tzinfo=datetime.timezone.utc) + else: + logger.warning("Unexpected DTSTART/DTEND type: dtstart=%s, dtend=%s", type(dtstart), type(dtend)) + continue if dtstart < time_range_end and dtend > time_range_start: - filtered_vevents.append(vevent) + if vevent not in filtered_vevents: # Avoid duplicates + logger.debug("VEVENT passed time-range filter: %s", dtstart) + filtered_vevents.append(vevent) + else: + logger.debug("VEVENT filtered out: %s", dtstart) - dtstart = vevent_recurrence.dtstart.value - dtend = vevent_recurrence.dtend.value if hasattr(vevent_recurrence, 'dtend') else dtstart - dtstart = datetime.datetime.strptime( - dtstart, "%Y%m%dT%H%M%SZ").replace( - tzinfo=datetime.timezone.utc) - dtend = datetime.datetime.strptime( - dtend, "%Y%m%dT%H%M%SZ").replace( - tzinfo=datetime.timezone.utc) - - if filtered_vevents or (dtstart < time_range_end and dtend > time_range_start): - if filtered_vevents: - vevent_component.vevent = filtered_vevents[0] - for vevent in filtered_vevents[1:]: - vevent_component.add(vevent) - if dtstart < time_range_end and dtend > time_range_start: - if not filtered_vevents: - vevent_component.vevent = vevent_recurrence - else: - vevent_component.add(vevent_recurrence) - else: - element.text = "" - return element + # Rebuild component + # ToDo: Get rid of return vevent_recurrence if filtered_vevents is empty it's wrong behavior + vevent_component.vevent_list = filtered_vevents if filtered_vevents else [vevent_recurrence] element.text = vevent_component.serialize() + logger.debug("Returning %d VEVENTs", len(vevent_component.vevent_list)) return element From 26215dbdbc8f0493d7e310c1b1c4cb12428938e5 Mon Sep 17 00:00:00 2001 From: Georgiy Date: Wed, 2 Jul 2025 23:34:08 +0300 Subject: [PATCH 05/35] (#1812) Fix typos --- radicale/app/report.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/radicale/app/report.py b/radicale/app/report.py index 1df045d0..38d8d719 100644 --- a/radicale/app/report.py +++ b/radicale/app/report.py @@ -366,7 +366,7 @@ def _expand( if hasattr(vevent_recurrence, 'exdate'): exdates = vevent_recurrence.exdate.value if not isinstance(exdates, list): - exdates = [exdates] # Convert single date to list + exdates = [exdates] logger.debug("EXDATE values: %s", exdates) latest_exdate = max(exdates) if exdates else None if latest_exdate and end > latest_exdate: @@ -451,7 +451,7 @@ def _expand( if all_day_event: dtend = dtend.date() except ValueError as e: - logger.warning("Invalid DTSTART format: %s, error: %s", dtstart, e) + logger.warning("Invalid DTEND format: %s, error: %s", dtend, e) continue # Convert to datetime for comparison From 371d5057de6a1f729d198ab738dd6e19c9e55099 Mon Sep 17 00:00:00 2001 From: Georgiy Date: Thu, 3 Jul 2025 22:39:57 +0300 Subject: [PATCH 06/35] (#1812) Filtering events with a full set of filters if there is no expand property. If there is an expand, then apply "filter (except time range) -> expand -> filter (only time range)" approach --- radicale/app/report.py | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/radicale/app/report.py b/radicale/app/report.py index 38d8d719..cfd76a42 100644 --- a/radicale/app/report.py +++ b/radicale/app/report.py @@ -223,18 +223,22 @@ def xml_report(base_prefix: str, path: str, xml_request: Optional[ET.Element], filters = ( root.findall(xmlutils.make_clark("C:filter")) + root.findall(xmlutils.make_clark("CR:filter"))) + expand = root.find(".//" + xmlutils.make_clark("C:expand")) - # extract time-range filter for processing after main filters + # if we have expand prop we use "filter (except time range) -> expand -> filter (only time range)" approach time_range_element = None - non_time_range_filters = [] + main_filters = [] for filter_ in filters: + # extract time-range filter for processing after main filters + # for expand request time_range_element = filter_.find(".//" + xmlutils.make_clark("C:time-range")) - if time_range_element is None: - non_time_range_filters.append(filter_) + + if expand is None or time_range_element is None: + main_filters.append(filter_) # Retrieve everything required for finishing the request. retrieved_items = list(retrieve_items( - base_prefix, path, collection, hreferences, non_time_range_filters, multistatus)) + base_prefix, path, collection, hreferences, main_filters, multistatus)) collection_tag = collection.tag # !!! Don't access storage after this !!! unlock_storage_fn() @@ -247,7 +251,7 @@ def xml_report(base_prefix: str, path: str, xml_request: Optional[ET.Element], if filters and not filters_matched: try: if not all(test_filter(collection_tag, item, filter_) - for filter_ in non_time_range_filters): + for filter_ in main_filters): continue except ValueError as e: raise ValueError("Failed to filter item %r from %r: %s" % @@ -257,7 +261,7 @@ def xml_report(base_prefix: str, path: str, xml_request: Optional[ET.Element], (item.href, collection.path, e)) from e # Filtering non-recurring events by time-range - if (time_range_element is not None) and not hasattr(item, 'rrule'): + if (expand is not None) and (time_range_element is not None) and not hasattr(item, 'rrule'): start, end = radicale_filter.time_range_timestamps(time_range_element) istart, iend = item.time_range if istart >= end or iend <= start: @@ -279,8 +283,7 @@ def xml_report(base_prefix: str, path: str, xml_request: Optional[ET.Element], xmlutils.make_clark("CR:address-data")): element.text = item.serialize() - expand = prop.find(xmlutils.make_clark("C:expand")) - if expand is not None and item.component_name == 'VEVENT': + if (expand is not None) and item.component_name == 'VEVENT': start = expand.get('start') end = expand.get('end') From 3afdc73bc64262a31d81e4eeebb139bae77fbf43 Mon Sep 17 00:00:00 2001 From: David Greaves Date: Thu, 3 Jul 2025 18:07:03 +0100 Subject: [PATCH 07/35] The item's rrule and time_range are in the vobject_item Signed-off-by: David Greaves --- radicale/app/report.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/radicale/app/report.py b/radicale/app/report.py index cfd76a42..979e949f 100644 --- a/radicale/app/report.py +++ b/radicale/app/report.py @@ -261,9 +261,15 @@ def xml_report(base_prefix: str, path: str, xml_request: Optional[ET.Element], (item.href, collection.path, e)) from e # Filtering non-recurring events by time-range - if (expand is not None) and (time_range_element is not None) and not hasattr(item, 'rrule'): + if hasattr(item.vobject_item, "vevent_list") and len(item.vobject_item.vevent_list) != 1: + # Not sure how to loop over these vevents + raise ValueError("vobject has too many events") + if ((expand is not None) and + (time_range_element is not None) and + not hasattr(item.vobject_item.vevent, + 'rrule')): start, end = radicale_filter.time_range_timestamps(time_range_element) - istart, iend = item.time_range + istart, iend = item.vobject_item.time_range if istart >= end or iend <= start: continue From 24b91e956914fc108bd6cc7d7eef35e1110b09e4 Mon Sep 17 00:00:00 2001 From: David Greaves Date: Fri, 4 Jul 2025 12:01:51 +0100 Subject: [PATCH 08/35] Remove the Filtering non-recurring events by time-range logic The tests pass 100% without this code and attempting to pre-filter these events here is quite complex and throws up test failures in multiple locations. Signed-off-by: David Greaves --- radicale/app/report.py | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/radicale/app/report.py b/radicale/app/report.py index 979e949f..334b6533 100644 --- a/radicale/app/report.py +++ b/radicale/app/report.py @@ -260,19 +260,6 @@ def xml_report(base_prefix: str, path: str, xml_request: Optional[ET.Element], raise RuntimeError("Failed to filter item %r from %r: %s" % (item.href, collection.path, e)) from e - # Filtering non-recurring events by time-range - if hasattr(item.vobject_item, "vevent_list") and len(item.vobject_item.vevent_list) != 1: - # Not sure how to loop over these vevents - raise ValueError("vobject has too many events") - if ((expand is not None) and - (time_range_element is not None) and - not hasattr(item.vobject_item.vevent, - 'rrule')): - start, end = radicale_filter.time_range_timestamps(time_range_element) - istart, iend = item.vobject_item.time_range - if istart >= end or iend <= start: - continue - found_props = [] not_found_props = [] From feb3daf25104f55e2a0f7b3e7a085fd381356f60 Mon Sep 17 00:00:00 2001 From: David Greaves Date: Thu, 3 Jul 2025 18:49:55 +0100 Subject: [PATCH 09/35] Find events which are in progress at the start of an expand range Signed-off-by: David Greaves --- radicale/app/report.py | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/radicale/app/report.py b/radicale/app/report.py index 334b6533..f1bfe73b 100644 --- a/radicale/app/report.py +++ b/radicale/app/report.py @@ -375,7 +375,19 @@ def _expand( filtered_vevents = [] if rruleset: # This function uses datetimes internally without timezone info for dates - recurrences = rruleset.between(start, end, inc=True) + + # A vobject rruleset is for the event dtstart. + # Expanded over a given time range this will not include + # events which started before the time range but are still + # ongoing at the start of the range + + # To accomodate this, reduce the start time by the duration of + # the event. If this introduces an extra reccurence point then + # that event should be included as it is still ongoing. If no + # extra point is generated then it was a no-op. + + rstart = start - duration if duration else start + recurrences = rruleset.between(rstart, end, inc=True) _strip_component(vevent_component) _strip_single_event(vevent_recurrence, dt_format) From 056954387d478b772c7ce01750bc8773d2d2c7bd Mon Sep 17 00:00:00 2001 From: David Greaves Date: Thu, 3 Jul 2025 19:29:15 +0100 Subject: [PATCH 10/35] fixes test_report_with_expand_property_all_day_event Signed-off-by: David Greaves --- radicale/app/report.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/radicale/app/report.py b/radicale/app/report.py index f1bfe73b..2601b453 100644 --- a/radicale/app/report.py +++ b/radicale/app/report.py @@ -402,7 +402,8 @@ def _expand( if time_range_start is not None and time_range_end is not None: dtstart = recurrence_utc dtend = dtstart + duration if duration else dtstart - if not (dtstart < time_range_end and dtend > time_range_start): + # Start includes the time, end does not + if not (dtstart <= time_range_end and dtend > time_range_start): logger.debug("Recurrence %s filtered out by time-range", recurrence_utc) continue From ec3366496e0a91d657f8a7b21d0826e5bc7d280e Mon Sep 17 00:00:00 2001 From: David Greaves Date: Thu, 3 Jul 2025 19:33:19 +0100 Subject: [PATCH 11/35] fixes tests where range starts inside an event test_report_with_expand_property_start_inside test_report_with_expand_property_just_inside Signed-off-by: David Greaves --- radicale/app/report.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/radicale/app/report.py b/radicale/app/report.py index 2601b453..89313c53 100644 --- a/radicale/app/report.py +++ b/radicale/app/report.py @@ -357,6 +357,8 @@ def _expand( duration = None if hasattr(vevent_recurrence, "dtend"): duration = vevent_recurrence.dtend.value - vevent_recurrence.dtstart.value + elif hasattr(vevent_recurrence, "duration"): + duration = vevent_recurrence.duration.value # Handle EXDATE to limit expansion range if hasattr(vevent_recurrence, 'exdate'): @@ -428,7 +430,8 @@ def _expand( name='DTSTART', value=recurrence_id.strftime(dt_format), params={} ) - if duration: + # if there is a DTEND, override it. Duration does not need changing + if hasattr(vevent, "dtend"): vevent.dtend = ContentLine( name='DTEND', value=(recurrence_id + duration).strftime(dt_format), params={} From 5c2e6377a22a5d5059853a95086b6afae29c2533 Mon Sep 17 00:00:00 2001 From: David Greaves Date: Fri, 4 Jul 2025 09:28:50 +0100 Subject: [PATCH 12/35] Don't limit expansion range using EXDATE Any EXDATE values specify single instances that must be removed from the expanded set of DTSTART values Signed-off-by: David Greaves --- radicale/app/report.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/radicale/app/report.py b/radicale/app/report.py index 89313c53..0e46185e 100644 --- a/radicale/app/report.py +++ b/radicale/app/report.py @@ -360,15 +360,13 @@ def _expand( elif hasattr(vevent_recurrence, "duration"): duration = vevent_recurrence.duration.value - # Handle EXDATE to limit expansion range + # Generate EXDATE to remove from expansion range if hasattr(vevent_recurrence, 'exdate'): exdates = vevent_recurrence.exdate.value if not isinstance(exdates, list): exdates = [exdates] logger.debug("EXDATE values: %s", exdates) - latest_exdate = max(exdates) if exdates else None - if latest_exdate and end > latest_exdate: - end = min(end, latest_exdate) + # TODO: these exdate values are not removed from the expanded set rruleset = None if hasattr(vevent_recurrence, 'rrule'): @@ -409,6 +407,8 @@ def _expand( logger.debug("Recurrence %s filtered out by time-range", recurrence_utc) continue + # Check here for exdate + # Check for overridden instances i_overridden, vevent = _find_overridden(i_overridden, vevents_overridden, recurrence_utc, dt_format) From 27fe8f1d634839756b6c50588d1a7e05c50310e8 Mon Sep 17 00:00:00 2001 From: David Greaves Date: Fri, 4 Jul 2025 10:14:09 +0100 Subject: [PATCH 13/35] Don't add the recurrence event here as we're expanding. This fixes test_report_with_expand_property_timezone Signed-off-by: David Greaves --- radicale/app/report.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/radicale/app/report.py b/radicale/app/report.py index 0e46185e..1dba7552 100644 --- a/radicale/app/report.py +++ b/radicale/app/report.py @@ -441,7 +441,7 @@ def _expand( # Filter overridden and recurrence base events if time_range_start is not None and time_range_end is not None: - for vevent in vevents_overridden + [vevent_recurrence]: + for vevent in vevents_overridden: dtstart = vevent.dtstart.value dtend = vevent.dtend.value if hasattr(vevent, 'dtend') else dtstart logger.debug( From 02be6419aec42e0f2de32ede305fa0b0b1e34c2a Mon Sep 17 00:00:00 2001 From: David Greaves Date: Thu, 3 Jul 2025 19:48:58 +0100 Subject: [PATCH 14/35] Currently passing tests Signed-off-by: David Greaves --- radicale/tests/test_expand.py | 26 ++++++++++++++++++++++---- 1 file changed, 22 insertions(+), 4 deletions(-) diff --git a/radicale/tests/test_expand.py b/radicale/tests/test_expand.py index 3055eafb..c9cee5f9 100644 --- a/radicale/tests/test_expand.py +++ b/radicale/tests/test_expand.py @@ -25,6 +25,9 @@ from typing import ClassVar, List from radicale.tests import BaseTest from radicale.tests.helpers import get_file_content +from radicale.log import logger + +from xml.etree import ElementTree ONLY_DATES = True CONTAINS_TIMES = False @@ -144,6 +147,8 @@ permissions: RrWw""") assert not isinstance(response_with_expand, int) status, element = response_with_expand["C:calendar-data"] + logger.debug("lbt: element is %s", + ElementTree.tostring(element, encoding='unicode')) assert status == 200 and element.text assert "RRULE" not in element.text assert "BEGIN:VTIMEZONE" not in element.text @@ -209,15 +214,28 @@ permissions: RrWw""") def test_report_with_expand_property_issue1812(self) -> None: """Test report with expand property for issue 1812""" + self._test_expand( + "event_issue1812", + "20250127T183000Z", + "20250127T183001Z", + ["RECURRENCE-ID:20250127T180000Z"], + ["DTSTART:20250127T180000Z"], + ["DTEND:20250127T233000Z"], + CONTAINS_TIMES, + 11 + ) + + def test_report_with_expand_property_issue1812_DS(self) -> None: + """Test report with expand property for issue 1812 - DS active""" self._test_expand( "event_issue1812", "20250627T183000Z", "20250627T183001Z", - ["RECURRENCE-ID:20250627T180000Z"], - ["DTSTART:20250627T180000Z"], - [], + ["RECURRENCE-ID:20250627T170000Z"], + ["DTSTART:20250627T170000Z"], + ["DTEND:20250627T223000Z"], CONTAINS_TIMES, - 1 + 11 ) def test_report_with_expand_property_all_day_event(self) -> None: From 2a304259fe28f14d575afeed0bf3b1ecd794dc4e Mon Sep 17 00:00:00 2001 From: Georgiy Date: Fri, 4 Jul 2025 22:47:05 +0300 Subject: [PATCH 15/35] (#1812) Processing exdates. Duration handling. Return empty element if all events was filtered Author: Georgiy --- radicale/app/report.py | 56 ++++++++++++++++++++++++++---------------- 1 file changed, 35 insertions(+), 21 deletions(-) diff --git a/radicale/app/report.py b/radicale/app/report.py index 1dba7552..ad176d63 100644 --- a/radicale/app/report.py +++ b/radicale/app/report.py @@ -358,15 +358,29 @@ def _expand( if hasattr(vevent_recurrence, "dtend"): duration = vevent_recurrence.dtend.value - vevent_recurrence.dtstart.value elif hasattr(vevent_recurrence, "duration"): - duration = vevent_recurrence.duration.value + try: + duration = vevent_recurrence.duration.value + if duration.total_seconds() <= 0: + logger.warning("Invalid DURATION: %s", duration) + duration = None + except (AttributeError, TypeError) as e: + logger.warning("Failed to parse DURATION: %s", e) + duration = None # Generate EXDATE to remove from expansion range + exdates_set = {} if hasattr(vevent_recurrence, 'exdate'): exdates = vevent_recurrence.exdate.value if not isinstance(exdates, list): exdates = [exdates] - logger.debug("EXDATE values: %s", exdates) - # TODO: these exdate values are not removed from the expanded set + + exdates_set = { + exdate.astimezone(datetime.timezone.utc) if isinstance(exdate, datetime.datetime) + else datetime.datetime.fromordinal(exdate.toordinal()).replace(tzinfo=None) + for exdate in exdates + } + + logger.debug("EXDATE values: %s", exdates_set) rruleset = None if hasattr(vevent_recurrence, 'rrule'): @@ -385,8 +399,7 @@ def _expand( # the event. If this introduces an extra reccurence point then # that event should be included as it is still ongoing. If no # extra point is generated then it was a no-op. - - rstart = start - duration if duration else start + rstart = start - duration if duration and duration.total_seconds() > 0 else start recurrences = rruleset.between(rstart, end, inc=True) _strip_component(vevent_component) @@ -407,7 +420,10 @@ def _expand( logger.debug("Recurrence %s filtered out by time-range", recurrence_utc) continue - # Check here for exdate + # Check exdate + if recurrence_utc in exdates_set: + logger.debug("Recurrence %s excluded by EXDATE", recurrence_utc) + continue # Check for overridden instances i_overridden, vevent = _find_overridden(i_overridden, vevents_overridden, recurrence_utc, dt_format) @@ -443,10 +459,6 @@ def _expand( if time_range_start is not None and time_range_end is not None: for vevent in vevents_overridden: dtstart = vevent.dtstart.value - dtend = vevent.dtend.value if hasattr(vevent, 'dtend') else dtstart - logger.debug( - "Filtering VEVENT with DTSTART: %s (type: %s), DTEND: %s (type: %s)", - dtstart, type(dtstart), dtend, type(dtend)) # Handle string values for DTSTART/DTEND if isinstance(dtstart, str): @@ -457,14 +469,12 @@ def _expand( except ValueError as e: logger.warning("Invalid DTSTART format: %s, error: %s", dtstart, e) continue - if isinstance(dtend, str): - try: - dtend = datetime.datetime.strptime(dtend, dt_format) - if all_day_event: - dtend = dtend.date() - except ValueError as e: - logger.warning("Invalid DTEND format: %s, error: %s", dtend, e) - continue + + dtend = dtstart + duration if duration else dtstart + + logger.debug( + "Filtering VEVENT with DTSTART: %s (type: %s), DTEND: %s (type: %s)", + dtstart, type(dtstart), dtend, type(dtend)) # Convert to datetime for comparison if all_day_event and isinstance(dtstart, datetime.date) and not isinstance(dtstart, datetime.datetime): @@ -487,10 +497,14 @@ def _expand( # Rebuild component - # ToDo: Get rid of return vevent_recurrence if filtered_vevents is empty it's wrong behavior - vevent_component.vevent_list = filtered_vevents if filtered_vevents else [vevent_recurrence] + if not filtered_vevents: + element.text = "" + return element + else: + vevent_component.vevent_list = filtered_vevents + logger.debug("lbt: vevent_component %s", vevent_component) + element.text = vevent_component.serialize() - logger.debug("Returning %d VEVENTs", len(vevent_component.vevent_list)) return element From ce0aaffa86fdab06040d68df9f8116da195f1fc0 Mon Sep 17 00:00:00 2001 From: David Greaves Date: Sun, 6 Jul 2025 16:51:57 +0100 Subject: [PATCH 16/35] fix typing and lint issues from tox Signed-off-by: David Greaves --- radicale/app/report.py | 12 ++++++------ radicale/tests/test_expand.py | 5 ++--- 2 files changed, 8 insertions(+), 9 deletions(-) diff --git a/radicale/app/report.py b/radicale/app/report.py index ad176d63..c24e17f4 100644 --- a/radicale/app/report.py +++ b/radicale/app/report.py @@ -277,18 +277,18 @@ def xml_report(base_prefix: str, path: str, xml_request: Optional[ET.Element], element.text = item.serialize() if (expand is not None) and item.component_name == 'VEVENT': - start = expand.get('start') - end = expand.get('end') + starts = expand.get('start') + ends = expand.get('end') - if (start is None) or (end is None): + if (starts is None) or (ends is None): return client.FORBIDDEN, \ xmlutils.webdav_error("C:expand") start = datetime.datetime.strptime( - start, '%Y%m%dT%H%M%SZ' + starts, '%Y%m%dT%H%M%SZ' ).replace(tzinfo=datetime.timezone.utc) end = datetime.datetime.strptime( - end, '%Y%m%dT%H%M%SZ' + ends, '%Y%m%dT%H%M%SZ' ).replace(tzinfo=datetime.timezone.utc) time_range_start = None @@ -368,7 +368,7 @@ def _expand( duration = None # Generate EXDATE to remove from expansion range - exdates_set = {} + exdates_set: set[datetime.datetime] = set() if hasattr(vevent_recurrence, 'exdate'): exdates = vevent_recurrence.exdate.value if not isinstance(exdates, list): diff --git a/radicale/tests/test_expand.py b/radicale/tests/test_expand.py index c9cee5f9..2b7dee1d 100644 --- a/radicale/tests/test_expand.py +++ b/radicale/tests/test_expand.py @@ -22,12 +22,11 @@ Radicale tests with expand requests. import os from typing import ClassVar, List +from xml.etree import ElementTree +from radicale.log import logger from radicale.tests import BaseTest from radicale.tests.helpers import get_file_content -from radicale.log import logger - -from xml.etree import ElementTree ONLY_DATES = True CONTAINS_TIMES = False From a26bfaa08aa6ebc52cca714c35d20cc2836d58b1 Mon Sep 17 00:00:00 2001 From: David Greaves Date: Mon, 7 Jul 2025 14:06:44 +0100 Subject: [PATCH 17/35] Add copyright for contributors Signed-off-by: David Greaves --- radicale/app/report.py | 3 ++- radicale/tests/test_expand.py | 2 ++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/radicale/app/report.py b/radicale/app/report.py index c24e17f4..8e84a9fc 100644 --- a/radicale/app/report.py +++ b/radicale/app/report.py @@ -5,8 +5,9 @@ # Copyright © 2017-2021 Unrud # Copyright © 2024-2024 Pieter Hijma # Copyright © 2024-2024 Ray -# Copyright © 2024-2024 Georgiy +# Copyright © 2024-2025 Georgiy # Copyright © 2024-2025 Peter Bieringer +# Copyright © 2025-2025 David Greaves # # This library is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by diff --git a/radicale/tests/test_expand.py b/radicale/tests/test_expand.py index 2b7dee1d..b4d54b22 100644 --- a/radicale/tests/test_expand.py +++ b/radicale/tests/test_expand.py @@ -1,6 +1,8 @@ # This file is part of Radicale - CalDAV and CardDAV server # Copyright © 2012-2017 Guillaume Ayoub # Copyright © 2017-2019 Unrud +# Copyright © 2024 Pieter Hijma +# Copyright © 2025 David Greaves # # This library is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by From fbc3abc48dd0cd35bcadff649fc899b722c9bd00 Mon Sep 17 00:00:00 2001 From: David Greaves Date: Mon, 7 Jul 2025 21:02:43 +0100 Subject: [PATCH 18/35] support max_occurence anti-DoS in xml_report for vevent expansion Signed-off-by: David Greaves --- radicale/app/report.py | 33 ++++++++++++++++++++++++--------- 1 file changed, 24 insertions(+), 9 deletions(-) diff --git a/radicale/app/report.py b/radicale/app/report.py index 8e84a9fc..4c56adb1 100644 --- a/radicale/app/report.py +++ b/radicale/app/report.py @@ -145,7 +145,8 @@ def free_busy_report(base_prefix: str, path: str, xml_request: Optional[ET.Eleme def xml_report(base_prefix: str, path: str, xml_request: Optional[ET.Element], collection: storage.BaseCollection, encoding: str, - unlock_storage_fn: Callable[[], None] + unlock_storage_fn: Callable[[], None], + max_occurrence: int = 0, ) -> Tuple[int, ET.Element]: """Read and answer REPORT requests that return XML. @@ -244,6 +245,7 @@ def xml_report(base_prefix: str, path: str, xml_request: Optional[ET.Element], # !!! Don't access storage after this !!! unlock_storage_fn() + n_vevents = 0 while retrieved_items: # ``item.vobject_item`` might be accessed during filtering. # Don't keep reference to ``item``, because VObject requires a lot of @@ -298,15 +300,21 @@ def xml_report(base_prefix: str, path: str, xml_request: Optional[ET.Element], if time_range_element is not None: time_range_start, time_range_end = radicale_filter.parse_time_range(time_range_element) - expanded_element = _expand( + (expanded_element, n_vev) = _expand( element=element, item=copy.copy(item), start=start, end=end, time_range_start=time_range_start, time_range_end=time_range_end, + max_occurrence=max_occurrence, ) - + n_vevents += n_vev found_props.append(expanded_element) else: found_props.append(element) + n_vevents += 1 + # Avoid DoS with too many events + if max_occurrence and n_vevents > max_occurrence: + raise ValueError("REPORT occurrences limit of {} hit" + .format(max_occurrence)) else: not_found_props.append(element) @@ -327,7 +335,8 @@ def _expand( end: datetime.datetime, time_range_start: Optional[datetime.datetime] = None, time_range_end: Optional[datetime.datetime] = None, -) -> ET.Element: + max_occurrence: int = 0, +) -> Tuple[ET.Element, int]: vevent_component: vobject.base.Component = copy.copy(item.vobject_item) logger.info("Expanding event %s", item.href) @@ -401,7 +410,13 @@ def _expand( # that event should be included as it is still ongoing. If no # extra point is generated then it was a no-op. rstart = start - duration if duration and duration.total_seconds() > 0 else start - recurrences = rruleset.between(rstart, end, inc=True) + recurrences = rruleset.between(rstart, end, inc=True, count=max_occurrence) + if max_occurrence and len(recurrences) >= max_occurrence: + # this shouldn't be > and if it's == then assume a limit + # was hit and ignore that maybe some would be filtered out + # by EXDATE etc. This is anti-DoS, not precise limits + raise ValueError("REPORT occurrences limit of {} hit" + .format(max_occurrence)) _strip_component(vevent_component) _strip_single_event(vevent_recurrence, dt_format) @@ -500,14 +515,14 @@ def _expand( if not filtered_vevents: element.text = "" - return element + return element, 0 else: vevent_component.vevent_list = filtered_vevents logger.debug("lbt: vevent_component %s", vevent_component) element.text = vevent_component.serialize() - return element + return element, len(filtered_vevents) def _convert_timezone(vevent: vobject.icalendar.RecurringComponent, @@ -744,9 +759,9 @@ class ApplicationPartReport(ApplicationBase): assert item.collection is not None collection = item.collection + max_occurrence = self.configuration.get("reporting", "max_freebusy_occurrence") if xml_content is not None and \ xml_content.tag == xmlutils.make_clark("C:free-busy-query"): - max_occurrence = self.configuration.get("reporting", "max_freebusy_occurrence") try: status, body = free_busy_report( base_prefix, path, xml_content, collection, self._encoding, @@ -761,7 +776,7 @@ class ApplicationPartReport(ApplicationBase): try: status, xml_answer = xml_report( base_prefix, path, xml_content, collection, self._encoding, - lock_stack.close) + lock_stack.close, max_occurrence) except ValueError as e: logger.warning( "Bad REPORT request on %r: %s", path, e, exc_info=True) From 61850d9b137d1877dc2498b024b2ea4bb0edf313 Mon Sep 17 00:00:00 2001 From: David Greaves Date: Tue, 8 Jul 2025 13:24:59 +0100 Subject: [PATCH 19/35] refactor _test_expand ready to support max_freebusy_occurrence Signed-off-by: David Greaves --- radicale/tests/test_expand.py | 76 ++++++++++++++++++++--------------- 1 file changed, 44 insertions(+), 32 deletions(-) diff --git a/radicale/tests/test_expand.py b/radicale/tests/test_expand.py index b4d54b22..0614bd91 100644 --- a/radicale/tests/test_expand.py +++ b/radicale/tests/test_expand.py @@ -72,17 +72,13 @@ permissions: RrWw""") self.configure({"rights": {"file": rights_file_path, "type": "from_file"}}) - def _test_expand(self, - expected_uid: str, - start: str, - end: str, - expected_recurrence_ids: List[str], - expected_start_times: List[str], - expected_end_times: List[str], - only_dates: bool, - nr_uids: int) -> None: + def _req_without_expand(self, + expected_uid: str, + start: str, + end: str, + ) -> str: self.put("/calendar.ics/", get_file_content(f"{expected_uid}.ics")) - req_body_without_expand = \ + return \ f""" @@ -98,9 +94,43 @@ permissions: RrWw""") """ - _, responses = self.report("/calendar.ics/", req_body_without_expand) - assert len(responses) == 1 + def _req_with_expand(self, + expected_uid: str, + start: str, + end: str, + ) -> str: + self.put("/calendar.ics/", get_file_content(f"{expected_uid}.ics")) + return \ + f""" + + + + + + + + + + + + + + + """ + + def _test_expand(self, + expected_uid: str, + start: str, + end: str, + expected_recurrence_ids: List[str], + expected_start_times: List[str], + expected_end_times: List[str], + only_dates: bool, + nr_uids: int) -> None: + _, responses = self.report("/calendar.ics/", + self._req_without_expand(expected_uid, start, end)) + assert len(responses) == 1 response_without_expand = responses[f'/calendar.ics/{expected_uid}.ics'] assert not isinstance(response_without_expand, int) status, element = response_without_expand["C:calendar-data"] @@ -122,26 +152,8 @@ permissions: RrWw""") assert len(uids) == nr_uids - req_body_with_expand = \ - f""" - - - - - - - - - - - - - - - """ - - _, responses = self.report("/calendar.ics/", req_body_with_expand) - + _, responses = self.report("/calendar.ics/", + self._req_with_expand(expected_uid, start, end)) assert len(responses) == 1 response_with_expand = responses[f'/calendar.ics/{expected_uid}.ics'] From 3f4f40555429dd64e93395904cf66cbb824ce824 Mon Sep 17 00:00:00 2001 From: David Greaves Date: Tue, 8 Jul 2025 15:08:44 +0100 Subject: [PATCH 20/35] Test for anti-DoS for expand The only response from the server is a 400 Signed-off-by: David Greaves --- .../static/event_daily_rrule_forever.ics | 28 +++++++++++++++ radicale/tests/test_expand.py | 36 ++++++++++++++++++- 2 files changed, 63 insertions(+), 1 deletion(-) create mode 100644 radicale/tests/static/event_daily_rrule_forever.ics diff --git a/radicale/tests/static/event_daily_rrule_forever.ics b/radicale/tests/static/event_daily_rrule_forever.ics new file mode 100644 index 00000000..c6a2970b --- /dev/null +++ b/radicale/tests/static/event_daily_rrule_forever.ics @@ -0,0 +1,28 @@ +BEGIN:VCALENDAR +VERSION:2.0 +BEGIN:VTIMEZONE +LAST-MODIFIED:20040110T032845Z +TZID:US/Eastern +BEGIN:DAYLIGHT +DTSTART:20000404T020000 +RRULE:FREQ=YEARLY;BYDAY=1SU;BYMONTH=4 +TZNAME:EDT +TZOFFSETFROM:-0500 +TZOFFSETTO:-0400 +END:DAYLIGHT +BEGIN:STANDARD +DTSTART:20001026T020000 +RRULE:FREQ=YEARLY;BYDAY=-1SU;BYMONTH=10 +TZNAME:EST +TZOFFSETFROM:-0400 +TZOFFSETTO:-0500 +END:STANDARD +END:VTIMEZONE +BEGIN:VEVENT +DTSTART;TZID=US/Eastern:20060102T120000 +DURATION:PT1H +RRULE:FREQ=DAILY +SUMMARY:Recurring event +UID:event_daily_rrule_forever +END:VEVENT +END:VCALENDAR diff --git a/radicale/tests/test_expand.py b/radicale/tests/test_expand.py index 0614bd91..3b353556 100644 --- a/radicale/tests/test_expand.py +++ b/radicale/tests/test_expand.py @@ -23,7 +23,7 @@ Radicale tests with expand requests. """ import os -from typing import ClassVar, List +from typing import ClassVar, List, Optional from xml.etree import ElementTree from radicale.log import logger @@ -154,6 +154,7 @@ permissions: RrWw""") _, responses = self.report("/calendar.ics/", self._req_with_expand(expected_uid, start, end)) + assert len(responses) == 1 response_with_expand = responses[f'/calendar.ics/{expected_uid}.ics'] @@ -186,6 +187,29 @@ permissions: RrWw""") assert len(uids) == len(expected_recurrence_ids) assert len(set(recurrence_ids)) == len(expected_recurrence_ids) + def _test_expand_max(self, + expected_uid: str, + start: str, + end: str, + check: Optional[int] = None) -> None: + _, responses = self.report("/calendar.ics/", + self._req_without_expand(expected_uid, start, end)) + assert len(responses) == 1 + response_without_expand = responses[f'/calendar.ics/{expected_uid}.ics'] + assert not isinstance(response_without_expand, int) + status, element = response_without_expand["C:calendar-data"] + + assert status == 200 and element.text + + assert "RRULE" in element.text + + status, headers, answer = self.request( + "REPORT", "/calendar.ics/", + self._req_with_expand(expected_uid, start, end), + check=check) + + assert status == 400 + def test_report_with_expand_property(self) -> None: """Test report with expand property""" self._test_expand( @@ -298,3 +322,13 @@ permissions: RrWw""") CONTAINS_TIMES, 1 ) + + def test_report_with_expand_property_max_occur(self) -> None: + """Test report with expand property too many vevents""" + self.configure({"reporting": {"max_freebusy_occurrence": 100}}) + self._test_expand_max( + "event_daily_rrule_forever", + "20060103T000000Z", + "20060501T000000Z", + check=400 + ) From 83a7853405d560438eb732b4a179224ef49ad7eb Mon Sep 17 00:00:00 2001 From: David Greaves Date: Tue, 8 Jul 2025 16:09:28 +0100 Subject: [PATCH 21/35] add test_report_with_max_occur and fix vevent count issue found Signed-off-by: David Greaves --- radicale/app/report.py | 2 +- .../tests/static/event_multiple_too_many.ics | 100 ++++++++++++++++++ radicale/tests/test_expand.py | 17 ++- 3 files changed, 117 insertions(+), 2 deletions(-) create mode 100644 radicale/tests/static/event_multiple_too_many.ics diff --git a/radicale/app/report.py b/radicale/app/report.py index 4c56adb1..26ad1722 100644 --- a/radicale/app/report.py +++ b/radicale/app/report.py @@ -310,7 +310,7 @@ def xml_report(base_prefix: str, path: str, xml_request: Optional[ET.Element], found_props.append(expanded_element) else: found_props.append(element) - n_vevents += 1 + n_vevents += len(item.vobject_item.vevent_list) # Avoid DoS with too many events if max_occurrence and n_vevents > max_occurrence: raise ValueError("REPORT occurrences limit of {} hit" diff --git a/radicale/tests/static/event_multiple_too_many.ics b/radicale/tests/static/event_multiple_too_many.ics new file mode 100644 index 00000000..4047ef06 --- /dev/null +++ b/radicale/tests/static/event_multiple_too_many.ics @@ -0,0 +1,100 @@ +BEGIN:VCALENDAR +PRODID:-//Mozilla.org/NONSGML Mozilla Calendar V1.1//EN +VERSION:2.0 +BEGIN:VTIMEZONE +TZID:Europe/Paris +X-LIC-LOCATION:Europe/Paris +BEGIN:DAYLIGHT +TZOFFSETFROM:+0100 +TZOFFSETTO:+0200 +TZNAME:CEST +DTSTART:19700329T020000 +RRULE:FREQ=YEARLY;BYDAY=-1SU;BYMONTH=3 +END:DAYLIGHT +BEGIN:STANDARD +TZOFFSETFROM:+0200 +TZOFFSETTO:+0100 +TZNAME:CET +DTSTART:19701025T030000 +RRULE:FREQ=YEARLY;BYDAY=-1SU;BYMONTH=10 +END:STANDARD +END:VTIMEZONE +BEGIN:VEVENT +UID:event_multiple_too_many +SUMMARY:Event +DTSTART;TZID=Europe/Paris:20130901T190000 +DTEND;TZID=Europe/Paris:20130901T200000 +END:VEVENT +BEGIN:VEVENT +UID:event_multiple_too_many1 +SUMMARY:Event1 +DTSTART;TZID=Europe/Paris:20130901T190000 +DTEND;TZID=Europe/Paris:20130901T200000 +END:VEVENT +BEGIN:VEVENT +UID:event_multiple_too_many2 +SUMMARY:Event2 +DTSTART;TZID=Europe/Paris:20130901T190000 +DTEND;TZID=Europe/Paris:20130901T200000 +END:VEVENT +BEGIN:VEVENT +UID:event_multiple_too_many3 +SUMMARY:Event3 +DTSTART;TZID=Europe/Paris:20130901T190000 +DTEND;TZID=Europe/Paris:20130901T200000 +END:VEVENT +BEGIN:VEVENT +UID:event_multiple_too_many4 +SUMMARY:Event4 +DTSTART;TZID=Europe/Paris:20130901T190000 +DTEND;TZID=Europe/Paris:20130901T200000 +END:VEVENT +BEGIN:VEVENT +UID:event_multiple_too_many5 +SUMMARY:Event5 +DTSTART;TZID=Europe/Paris:20130901T190000 +DTEND;TZID=Europe/Paris:20130901T200000 +END:VEVENT +BEGIN:VEVENT +UID:event_multiple_too_many6 +SUMMARY:Event6 +DTSTART;TZID=Europe/Paris:20130901T190000 +DTEND;TZID=Europe/Paris:20130901T200000 +END:VEVENT +BEGIN:VEVENT +UID:event_multiple_too_many7 +SUMMARY:Event7 +DTSTART;TZID=Europe/Paris:20130901T190000 +DTEND;TZID=Europe/Paris:20130901T200000 +END:VEVENT +BEGIN:VEVENT +UID:event_multiple_too_many8 +SUMMARY:Event8 +DTSTART;TZID=Europe/Paris:20130901T190000 +DTEND;TZID=Europe/Paris:20130901T200000 +END:VEVENT +BEGIN:VEVENT +UID:event_multiple_too_many9 +SUMMARY:Event9 +DTSTART;TZID=Europe/Paris:20130901T190000 +DTEND;TZID=Europe/Paris:20130901T200000 +END:VEVENT +BEGIN:VEVENT +UID:event_multiple_too_many10 +SUMMARY:Event10 +DTSTART;TZID=Europe/Paris:20130901T190000 +DTEND;TZID=Europe/Paris:20130901T200000 +END:VEVENT +BEGIN:VEVENT +UID:event_multiple_too_many11 +SUMMARY:Event11 +DTSTART;TZID=Europe/Paris:20130901T190000 +DTEND;TZID=Europe/Paris:20130901T200000 +END:VEVENT +BEGIN:VTODO +UID:todo +DTSTART;TZID=Europe/Paris:20130901T220000 +DURATION:PT1H +SUMMARY:Todo +END:VTODO +END:VCALENDAR diff --git a/radicale/tests/test_expand.py b/radicale/tests/test_expand.py index 3b353556..bedc9d12 100644 --- a/radicale/tests/test_expand.py +++ b/radicale/tests/test_expand.py @@ -203,7 +203,7 @@ permissions: RrWw""") assert "RRULE" in element.text - status, headers, answer = self.request( + status, _, _ = self.request( "REPORT", "/calendar.ics/", self._req_with_expand(expected_uid, start, end), check=check) @@ -332,3 +332,18 @@ permissions: RrWw""") "20060501T000000Z", check=400 ) + + def test_report_with_max_occur(self) -> None: + """Test report with too many vevents""" + self.configure({"reporting": {"max_freebusy_occurrence": 10}}) + + uid = "event_multiple_too_many" + start = "20130901T000000Z" + end = "20130902T000000Z" + check = 400 + + status, responses = self.report("/calendar.ics/", + self._req_without_expand(uid, start, end), + check=check) + assert len(responses) == 0 + assert status == check From b61ab9d88631c164db02adac53c9e6239679ea60 Mon Sep 17 00:00:00 2001 From: Peter Bieringer Date: Thu, 10 Jul 2025 22:08:26 +0200 Subject: [PATCH 22/35] Changelog for https://github.com/Kozea/Radicale/pull/1815 --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 07e380e6..f78f5d6a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,7 @@ * Improve: display owner+permissions on directories on startup, extend error message in case of missing permissions * Feature: add hook for server-side e-mail notification * Fix: logging ignores not retrievable get_native_id if not supported by OS +* Fix: report with enabled expand honors now provided filter proper ## 3.5.4 * Improve: item filter enhanced for 3rd level supporting VALARM and honoring TRIGGER (offset or absolute) From a1280dc17c8739192e7c6e6a09a255eebd716e2e Mon Sep 17 00:00:00 2001 From: David Greaves Date: Sun, 13 Jul 2025 18:34:01 +0100 Subject: [PATCH 23/35] Guard against entries with no vevent_list in the vobject Signed-off-by: David Greaves --- radicale/app/report.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/radicale/app/report.py b/radicale/app/report.py index 26ad1722..44e41d1e 100644 --- a/radicale/app/report.py +++ b/radicale/app/report.py @@ -310,7 +310,8 @@ def xml_report(base_prefix: str, path: str, xml_request: Optional[ET.Element], found_props.append(expanded_element) else: found_props.append(element) - n_vevents += len(item.vobject_item.vevent_list) + if hasattr(item.vobject_item, "vevent_list"): + n_vevents += len(item.vobject_item.vevent_list) # Avoid DoS with too many events if max_occurrence and n_vevents > max_occurrence: raise ValueError("REPORT occurrences limit of {} hit" From 68f8d61bdeefc55d8570c7da1658daeb80426718 Mon Sep 17 00:00:00 2001 From: Georgiy Date: Mon, 14 Jul 2025 18:23:28 +0300 Subject: [PATCH 24/35] Skip items with no events after expansion to prevent empty in responses. Enhance component filtering --- radicale/app/report.py | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/radicale/app/report.py b/radicale/app/report.py index 44e41d1e..03f17e88 100644 --- a/radicale/app/report.py +++ b/radicale/app/report.py @@ -238,6 +238,18 @@ def xml_report(base_prefix: str, path: str, xml_request: Optional[ET.Element], if expand is None or time_range_element is None: main_filters.append(filter_) + # Extract requested component types from filters + requested_components = set() + has_vcalendar_filter = False + for filter_ in filters: + for comp_filter in filter_.findall(".//" + xmlutils.make_clark("C:comp-filter")): + component_name = comp_filter.get("name") + if component_name: + if component_name == "VCALENDAR": + has_vcalendar_filter = True + else: + requested_components.add(component_name) + # Retrieve everything required for finishing the request. retrieved_items = list(retrieve_items( base_prefix, path, collection, hreferences, main_filters, multistatus)) @@ -263,6 +275,13 @@ def xml_report(base_prefix: str, path: str, xml_request: Optional[ET.Element], raise RuntimeError("Failed to filter item %r from %r: %s" % (item.href, collection.path, e)) from e + # Skip items that don't match requested component types, unless VCALENDAR filter allows all components + if requested_components and not has_vcalendar_filter: + if item.component_name not in requested_components: + logger.debug("Skipping component %r (type: %s) as it doesn't match requested components %s", + item.href, item.component_name, requested_components) + continue + found_props = [] not_found_props = [] @@ -306,6 +325,11 @@ def xml_report(base_prefix: str, path: str, xml_request: Optional[ET.Element], time_range_start=time_range_start, time_range_end=time_range_end, max_occurrence=max_occurrence, ) + + if n_vev == 0: + logger.debug("No VEVENTs found after expansion for %r, skipping", item.href) + continue + n_vevents += n_vev found_props.append(expanded_element) else: From a76f80a62081d869641a62b08d288f056771af17 Mon Sep 17 00:00:00 2001 From: Georgiy Date: Wed, 16 Jul 2025 22:31:58 +0300 Subject: [PATCH 25/35] Separate time-range filter only for VEVENT component --- radicale/app/report.py | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/radicale/app/report.py b/radicale/app/report.py index 03f17e88..8f77c97b 100644 --- a/radicale/app/report.py +++ b/radicale/app/report.py @@ -228,22 +228,25 @@ def xml_report(base_prefix: str, path: str, xml_request: Optional[ET.Element], expand = root.find(".//" + xmlutils.make_clark("C:expand")) # if we have expand prop we use "filter (except time range) -> expand -> filter (only time range)" approach - time_range_element = None + vevent_time_range = None main_filters = [] for filter_ in filters: - # extract time-range filter for processing after main filters - # for expand request - time_range_element = filter_.find(".//" + xmlutils.make_clark("C:time-range")) + if expand is not None: + for comp_filter in filter_.findall(".//" + xmlutils.make_clark("C:comp-filter")): + if comp_filter.get("name", "").upper() == "VEVENT": + vevent_time_range = comp_filter.find(".//" + xmlutils.make_clark("C:time-range")) + if vevent_time_range is not None: + comp_filter.remove(vevent_time_range) + break - if expand is None or time_range_element is None: - main_filters.append(filter_) + main_filters.append(filter_) # Extract requested component types from filters requested_components = set() has_vcalendar_filter = False for filter_ in filters: for comp_filter in filter_.findall(".//" + xmlutils.make_clark("C:comp-filter")): - component_name = comp_filter.get("name") + component_name = comp_filter.get("name", "").upper() if component_name: if component_name == "VCALENDAR": has_vcalendar_filter = True @@ -316,8 +319,8 @@ def xml_report(base_prefix: str, path: str, xml_request: Optional[ET.Element], time_range_start = None time_range_end = None - if time_range_element is not None: - time_range_start, time_range_end = radicale_filter.parse_time_range(time_range_element) + if vevent_time_range is not None: + time_range_start, time_range_end = radicale_filter.parse_time_range(vevent_time_range) (expanded_element, n_vev) = _expand( element=element, item=copy.copy(item), From 40217a1360162031b68c68d64d570ca92dde5e9d Mon Sep 17 00:00:00 2001 From: Georgiy Date: Wed, 16 Jul 2025 23:26:40 +0300 Subject: [PATCH 26/35] Return empty REPORT response if no items found --- radicale/app/report.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/radicale/app/report.py b/radicale/app/report.py index 8f77c97b..c58e8e78 100644 --- a/radicale/app/report.py +++ b/radicale/app/report.py @@ -349,9 +349,11 @@ def xml_report(base_prefix: str, path: str, xml_request: Optional[ET.Element], assert item.href uri = pathutils.unstrip_path( posixpath.join(collection.path, item.href)) - multistatus.append(xml_item_response( - base_prefix, uri, found_props=found_props, - not_found_props=not_found_props, found_item=True)) + + if found_props or not_found_props: + multistatus.append(xml_item_response( + base_prefix, uri, found_props=found_props, + not_found_props=not_found_props, found_item=True)) return client.MULTI_STATUS, multistatus From 76188c210e1d358938c3c667678cd3b1d55a17ff Mon Sep 17 00:00:00 2001 From: Georgiy Date: Thu, 17 Jul 2025 09:03:55 +0300 Subject: [PATCH 27/35] Revert time_range processing --- radicale/app/report.py | 19 ++++++++----------- 1 file changed, 8 insertions(+), 11 deletions(-) diff --git a/radicale/app/report.py b/radicale/app/report.py index c58e8e78..3af9422a 100644 --- a/radicale/app/report.py +++ b/radicale/app/report.py @@ -228,18 +228,15 @@ def xml_report(base_prefix: str, path: str, xml_request: Optional[ET.Element], expand = root.find(".//" + xmlutils.make_clark("C:expand")) # if we have expand prop we use "filter (except time range) -> expand -> filter (only time range)" approach - vevent_time_range = None + time_range_element = None main_filters = [] for filter_ in filters: - if expand is not None: - for comp_filter in filter_.findall(".//" + xmlutils.make_clark("C:comp-filter")): - if comp_filter.get("name", "").upper() == "VEVENT": - vevent_time_range = comp_filter.find(".//" + xmlutils.make_clark("C:time-range")) - if vevent_time_range is not None: - comp_filter.remove(vevent_time_range) - break + # extract time-range filter for processing after main filters + # for expand request + time_range_element = filter_.find(".//" + xmlutils.make_clark("C:time-range")) - main_filters.append(filter_) + if expand is None or time_range_element is None: + main_filters.append(filter_) # Extract requested component types from filters requested_components = set() @@ -319,8 +316,8 @@ def xml_report(base_prefix: str, path: str, xml_request: Optional[ET.Element], time_range_start = None time_range_end = None - if vevent_time_range is not None: - time_range_start, time_range_end = radicale_filter.parse_time_range(vevent_time_range) + if time_range_element is not None: + time_range_start, time_range_end = radicale_filter.parse_time_range(time_range_element) (expanded_element, n_vev) = _expand( element=element, item=copy.copy(item), From e94aa1e7cfa86affaadfad8d64b2cc312f01f435 Mon Sep 17 00:00:00 2001 From: Georgiy Date: Thu, 17 Jul 2025 16:26:21 +0300 Subject: [PATCH 28/35] Tests for comp-filters and not found events --- radicale/tests/test_expand.py | 111 ++++++++++++++++++++++++++++++++++ 1 file changed, 111 insertions(+) diff --git a/radicale/tests/test_expand.py b/radicale/tests/test_expand.py index bedc9d12..faa81078 100644 --- a/radicale/tests/test_expand.py +++ b/radicale/tests/test_expand.py @@ -347,3 +347,114 @@ permissions: RrWw""") check=check) assert len(responses) == 0 assert status == check + + def test_report_vcalendar_all_components(self) -> None: + """Test calendar-query with comp-filter VCALENDAR, returns all components.""" + self.mkcalendar("/test/") + self.put("/test/calendar.ics", get_file_content("event_daily_rrule.ics")) + self.put("/test/todo.ics", get_file_content("todo1.ics")) + + request = """ + + + + + + + + + """ + status, responses = self.report("/test", request) + assert status == 207 + assert len(responses) == 2 + assert "/test/calendar.ics" in responses + assert "/test/todo.ics" in responses + + def test_report_vevent_only(self) -> None: + """Test calendar-query with comp-filter VEVENT, returns only VEVENT.""" + self.mkcalendar("/test/") + self.put("/test/calendar.ics", get_file_content("event_daily_rrule.ics")) + self.put("/test/todo.ics", get_file_content("todo1.ics")) + + request = """ + + + + + + + + + + + """ + status, responses = self.report("/test", request) + assert status == 207 + assert len(responses) == 1 + assert "/test/calendar.ics" in responses + vevent_response = responses["/test/calendar.ics"] + status, element = vevent_response["C:calendar-data"] + assert status == 200 and element.text + assert "BEGIN:VEVENT" in element.text + assert "UID:" in element.text + assert "BEGIN:VTODO" not in element.text + + def test_report_time_range_no_vevent(self) -> None: + """Test calendar-query with time-range filter, no matching VEVENT.""" + self.mkcalendar("/test/") + self.put("/test/calendar.ics/", get_file_content("event_daily_rrule.ics")) + + request = """ + + + + + + + + + + + + + + + """ + status, responses = self.report("/test", request) + assert status == 207 + assert len(responses) == 0 + + def test_report_time_range_one_vevent(self) -> None: + """Test calendar-query with time-range filter, matches one VEVENT.""" + self.mkcalendar("/test/") + self.put("/test/calendar1.ics/", get_file_content("event_daily_rrule.ics")) + self.put("/test/calendar2.ics/", get_file_content("event1.ics")) + + start = "20060101T000000Z" + end = "20060104T000000Z" + + request = f""" + + + + + + + + + + + + + + + """ + status, responses = self.report("/test", request) + assert status == 207 + assert len(responses) == 1 + response = responses["/test/calendar1.ics"] + status, element = response["C:calendar-data"] + assert status == 200 and element.text + assert "BEGIN:VEVENT" in element.text + assert "RECURRENCE-ID:20060103T170000Z" in element.text + assert "DTSTART:20060103T170000Z" in element.text From 3a6c72e93a4bd4ec2b547ea160a5f0b1d2122cea Mon Sep 17 00:00:00 2001 From: Georgiy Date: Thu, 17 Jul 2025 16:34:38 +0300 Subject: [PATCH 29/35] pep fix --- radicale/tests/test_expand.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/radicale/tests/test_expand.py b/radicale/tests/test_expand.py index faa81078..16ca3855 100644 --- a/radicale/tests/test_expand.py +++ b/radicale/tests/test_expand.py @@ -393,6 +393,7 @@ permissions: RrWw""") assert len(responses) == 1 assert "/test/calendar.ics" in responses vevent_response = responses["/test/calendar.ics"] + assert type(vevent_response) is dict status, element = vevent_response["C:calendar-data"] assert status == 200 and element.text assert "BEGIN:VEVENT" in element.text @@ -453,6 +454,7 @@ permissions: RrWw""") assert status == 207 assert len(responses) == 1 response = responses["/test/calendar1.ics"] + assert type(response) is dict status, element = response["C:calendar-data"] assert status == 200 and element.text assert "BEGIN:VEVENT" in element.text From 2c4cd321326bb542c138e3db2a098e293e94f2db Mon Sep 17 00:00:00 2001 From: Georgiy Date: Fri, 18 Jul 2025 16:37:49 +0300 Subject: [PATCH 30/35] Fixed extraction of time-range filter from request when processing expand property --- radicale/app/report.py | 32 +++++++++++--------------------- radicale/tests/test_expand.py | 13 ++++++++++--- 2 files changed, 21 insertions(+), 24 deletions(-) diff --git a/radicale/app/report.py b/radicale/app/report.py index 3af9422a..8cb1e9b2 100644 --- a/radicale/app/report.py +++ b/radicale/app/report.py @@ -233,22 +233,17 @@ def xml_report(base_prefix: str, path: str, xml_request: Optional[ET.Element], for filter_ in filters: # extract time-range filter for processing after main filters # for expand request - time_range_element = filter_.find(".//" + xmlutils.make_clark("C:time-range")) + filter_copy = copy.deepcopy(filter_) - if expand is None or time_range_element is None: - main_filters.append(filter_) + if expand is not None: + for comp_filter in filter_copy.findall(".//" + xmlutils.make_clark("C:comp-filter")): + if comp_filter.get("name", "").upper() == "VCALENDAR": + continue + time_range_element = comp_filter.find(xmlutils.make_clark("C:time-range")) + if time_range_element is not None: + comp_filter.remove(time_range_element) - # Extract requested component types from filters - requested_components = set() - has_vcalendar_filter = False - for filter_ in filters: - for comp_filter in filter_.findall(".//" + xmlutils.make_clark("C:comp-filter")): - component_name = comp_filter.get("name", "").upper() - if component_name: - if component_name == "VCALENDAR": - has_vcalendar_filter = True - else: - requested_components.add(component_name) + main_filters.append(filter_copy) # Retrieve everything required for finishing the request. retrieved_items = list(retrieve_items( @@ -275,13 +270,6 @@ def xml_report(base_prefix: str, path: str, xml_request: Optional[ET.Element], raise RuntimeError("Failed to filter item %r from %r: %s" % (item.href, collection.path, e)) from e - # Skip items that don't match requested component types, unless VCALENDAR filter allows all components - if requested_components and not has_vcalendar_filter: - if item.component_name not in requested_components: - logger.debug("Skipping component %r (type: %s) as it doesn't match requested components %s", - item.href, item.component_name, requested_components) - continue - found_props = [] not_found_props = [] @@ -366,6 +354,8 @@ def _expand( ) -> Tuple[ET.Element, int]: vevent_component: vobject.base.Component = copy.copy(item.vobject_item) logger.info("Expanding event %s", item.href) + logger.debug(f"Expand range: {start} to {end}") + logger.debug(f"Time range: {time_range_start} to {time_range_end}") # Split the vevents included in the component into one that contains the # recurrence information and others that contain a recurrence id to diff --git a/radicale/tests/test_expand.py b/radicale/tests/test_expand.py index 16ca3855..2b10110d 100644 --- a/radicale/tests/test_expand.py +++ b/radicale/tests/test_expand.py @@ -376,14 +376,21 @@ permissions: RrWw""") self.put("/test/calendar.ics", get_file_content("event_daily_rrule.ics")) self.put("/test/todo.ics", get_file_content("todo1.ics")) - request = """ + start = "20060101T000000Z" + end = "20060104T000000Z" + + request = f""" - + + + - + + + From 02471b6c909e57aeb2c6d9a08b2df384fa419c69 Mon Sep 17 00:00:00 2001 From: Peter Bieringer Date: Sat, 19 Jul 2025 14:39:47 +0200 Subject: [PATCH 31/35] add trace options --- DOCUMENTATION.md | 18 ++++++++++++++++++ config | 6 ++++++ radicale/config.py | 8 ++++++++ 3 files changed, 32 insertions(+) diff --git a/DOCUMENTATION.md b/DOCUMENTATION.md index be7fe021..d00cd01a 100644 --- a/DOCUMENTATION.md +++ b/DOCUMENTATION.md @@ -1420,6 +1420,24 @@ Available levels: **debug**, **info**, **warning**, **error**, **critical** Default: `warning` _(< 3.2.0)_ `info` _(>= 3.2.0)_ +##### trace_on_debug + +_(> 3.5.4)_ + +Do not filter debug messages starting with 'TRACE' + +Default: `False` + +##### trace_filter + +_(> 3.5.4)_ + +Filter debug messages starting with 'TRACE/' + +Precondition: `trace_on_debug = True` + +Default: (empty) + ##### mask_passwords Don't include passwords in logs. diff --git a/config b/config index ab2ea7eb..51e5324c 100644 --- a/config +++ b/config @@ -272,6 +272,12 @@ # Value: debug | info | warning | error | critical #level = info +# do not filter debug messages starting with 'TRACE' +#trace_on_debug = False + +# filter debug messages starting with 'TRACE/' +#trace_filter = "" + # Don't include passwords in logs #mask_passwords = True diff --git a/radicale/config.py b/radicale/config.py index 15405063..7deee5ca 100644 --- a/radicale/config.py +++ b/radicale/config.py @@ -509,6 +509,14 @@ This is an automated message. Please do not reply.""", "value": "info", "help": "threshold for the logger", "type": logging_level}), + ("trace_on_debug", { + "value": "False", + "help": "do not filter debug messages starting with 'TRACE'", + "type": bool}), + ("trace_filter", { + "value": "", + "help": "filter debug messages starting with 'TRACE/'", + "type": str}), ("bad_put_request_content", { "value": "False", "help": "log bad PUT request content", From ef1c2b835fc5f2a5a290b49167f2f99be4c0afb6 Mon Sep 17 00:00:00 2001 From: Peter Bieringer Date: Sat, 19 Jul 2025 14:40:12 +0200 Subject: [PATCH 32/35] implement trace option --- radicale/__init__.py | 5 ++++- radicale/__main__.py | 5 ++++- radicale/log.py | 40 +++++++++++++++++++++++++++++++++++++++- 3 files changed, 47 insertions(+), 3 deletions(-) diff --git a/radicale/__init__.py b/radicale/__init__.py index 2554e5b2..8924d507 100644 --- a/radicale/__init__.py +++ b/radicale/__init__.py @@ -52,7 +52,10 @@ def _get_application_instance(config_path: str, wsgi_errors: types.ErrorStream configuration = config.load(config.parse_compound_paths( config.DEFAULT_CONFIG_PATH, config_path)) - log.set_level(cast(str, configuration.get("logging", "level")), configuration.get("logging", "backtrace_on_debug")) + log.set_level(cast(str, configuration.get("logging", "level")), + configuration.get("logging", "backtrace_on_debug"), + configuration.get("logging", "trace_on_debug"), + configuration.get("logging", "trace_filter")) # Log configuration after logger is configured default_config_active = True for source, miss in configuration.sources(): diff --git a/radicale/__main__.py b/radicale/__main__.py index 25d2b853..b3576a60 100644 --- a/radicale/__main__.py +++ b/radicale/__main__.py @@ -165,7 +165,10 @@ def run() -> None: sys.exit(1) # Configure logging - log.set_level(cast(str, configuration.get("logging", "level")), configuration.get("logging", "backtrace_on_debug")) + log.set_level(cast(str, configuration.get("logging", "level")), + configuration.get("logging", "backtrace_on_debug"), + configuration.get("logging", "trace_on_debug"), + configuration.get("logging", "trace_filter")) # Log configuration after logger is configured default_config_active = True diff --git a/radicale/log.py b/radicale/log.py index 1dde25d2..7f7df5e3 100644 --- a/radicale/log.py +++ b/radicale/log.py @@ -57,8 +57,35 @@ class RemoveTracebackFilter(logging.Filter): return True +class RemoveTRACEFilter(logging.Filter): + + def filter(self, record: logging.LogRecord) -> bool: + if record.msg.startswith("TRACE"): + return False + else: + return True + + +class PassTRACETOKENFilter(logging.Filter): + def __init__(self, trace_filter: str): + super().__init__() + self.trace_filter = trace_filter + self.prefix = "TRACE/" + self.trace_filter + + def filter(self, record: logging.LogRecord) -> bool: + if record.msg.startswith("TRACE"): + if record.msg.startswith(self.prefix): + return True + else: + return False + else: + return True + + REMOVE_TRACEBACK_FILTER: logging.Filter = RemoveTracebackFilter() +REMOVE_TRACE_FILTER: logging.Filter = RemoveTRACEFilter() + class IdentLogRecordFactory: """LogRecordFactory that adds ``ident`` attribute.""" @@ -231,7 +258,7 @@ logger_display_backtrace_disabled: bool = False logger_display_backtrace_enabled: bool = False -def set_level(level: Union[int, str], backtrace_on_debug: bool) -> None: +def set_level(level: Union[int, str], backtrace_on_debug: bool, trace_on_debug: bool = False, trace_filter: str = "") -> None: """Set logging level for global logger.""" global logger_display_backtrace_disabled global logger_display_backtrace_enabled @@ -255,3 +282,14 @@ def set_level(level: Union[int, str], backtrace_on_debug: bool) -> None: logger.debug("Logging of backtrace is enabled by option in this loglevel") logger_display_backtrace_enabled = True logger.removeFilter(REMOVE_TRACEBACK_FILTER) + if trace_on_debug: + if trace_filter != "": + logger.debug("Logging messages starting with 'TRACE/%s' enabled", trace_filter) + logger.addFilter(PassTRACETOKENFilter(trace_filter)) + logger.removeFilter(REMOVE_TRACE_FILTER) + else: + logger.debug("Logging messages starting with 'TRACE' enabled") + logger.removeFilter(REMOVE_TRACE_FILTER) + else: + logger.debug("Logging messages starting with 'TRACE' disabled") + logger.addFilter(REMOVE_TRACE_FILTER) From d5b589a5720027bd763a10560f0ab0302afedac0 Mon Sep 17 00:00:00 2001 From: Peter Bieringer Date: Sat, 19 Jul 2025 14:40:27 +0200 Subject: [PATCH 33/35] show trace status on startup --- radicale/server.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/radicale/server.py b/radicale/server.py index e437286a..55e112e2 100644 --- a/radicale/server.py +++ b/radicale/server.py @@ -340,6 +340,8 @@ def serve(configuration: config.Configuration, select_timeout = 1.0 max_connections: int = configuration.get("server", "max_connections") logger.info("Radicale server ready") + logger.debug("TRACE: Radicale server ready ('logging/trace_on_debug' is active)") + logger.debug("TRACE/SERVER: Radicale server ready ('logging/trace_on_debug' is active - either with 'SERVER' or empty filter)") while True: rlist: List[socket.socket] = [] # Wait for finished clients From be6fcd4b8a6d2737c166bf6fb64b45c2557324ae Mon Sep 17 00:00:00 2001 From: Peter Bieringer Date: Sat, 19 Jul 2025 14:42:18 +0200 Subject: [PATCH 34/35] extend changelog related to trace --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index f78f5d6a..ec1b5eeb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,7 @@ * Feature: add hook for server-side e-mail notification * Fix: logging ignores not retrievable get_native_id if not supported by OS * Fix: report with enabled expand honors now provided filter proper +* Improve: add options [logging] trace_on_debug and trace_filter for supporting trace logging ## 3.5.4 * Improve: item filter enhanced for 3rd level supporting VALARM and honoring TRIGGER (offset or absolute) From f31f5779f547833029ed2dd8569abe7b7ffc3a5c Mon Sep 17 00:00:00 2001 From: Peter Bieringer Date: Sat, 19 Jul 2025 14:46:46 +0200 Subject: [PATCH 35/35] add pytestdebug.log --- .gitignore | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.gitignore b/.gitignore index 5a0363a0..87b83643 100644 --- a/.gitignore +++ b/.gitignore @@ -23,3 +23,5 @@ coverage.xml .vscode .sass-cache Gemfile.lock + +pytestdebug.log