From ae731290c1ef697123bcd6f0a3ba07c78ed34f4a Mon Sep 17 00:00:00 2001 From: Georgiy Date: Thu, 30 Mar 2023 19:30:59 +0300 Subject: [PATCH 1/7] processing expand property for REPORT --- radicale/app/report.py | 74 ++++++++++++++++++++++++++++++++++++------ 1 file changed, 64 insertions(+), 10 deletions(-) diff --git a/radicale/app/report.py b/radicale/app/report.py index 5807f6e6..d5c94f8c 100644 --- a/radicale/app/report.py +++ b/radicale/app/report.py @@ -18,11 +18,16 @@ # along with Radicale. If not, see . import contextlib +import datetime import posixpath import socket +import copy import xml.etree.ElementTree as ET from http import client -from typing import Callable, Iterable, Iterator, Optional, Sequence, Tuple +from typing import ( + Callable, Iterable, Iterator, + Optional, Sequence, Tuple, List, +) from urllib.parse import unquote, urlparse import radicale.item as radicale_item @@ -64,9 +69,8 @@ def xml_report(base_prefix: str, path: str, xml_request: Optional[ET.Element], logger.warning("Invalid REPORT method %r on %r requested", xmlutils.make_human_tag(root.tag), path) return client.FORBIDDEN, xmlutils.webdav_error("D:supported-report") - prop_element = root.find(xmlutils.make_clark("D:prop")) - props = ([prop.tag for prop in prop_element] - if prop_element is not None else []) + + props = root.find(xmlutils.make_clark("D:prop")) or [] hreferences: Iterable[str] if root.tag in ( @@ -138,19 +142,40 @@ def xml_report(base_prefix: str, path: str, xml_request: Optional[ET.Element], found_props = [] not_found_props = [] - for tag in props: - element = ET.Element(tag) - if tag == xmlutils.make_clark("D:getetag"): + for prop in props: + element = ET.Element(prop.tag) + if prop.tag == xmlutils.make_clark("D:getetag"): element.text = item.etag found_props.append(element) - elif tag == xmlutils.make_clark("D:getcontenttype"): + elif prop.tag == xmlutils.make_clark("D:getcontenttype"): element.text = xmlutils.get_content_type(item, encoding) found_props.append(element) - elif tag in ( + elif prop.tag in ( xmlutils.make_clark("C:calendar-data"), xmlutils.make_clark("CR:address-data")): element.text = item.serialize() - found_props.append(element) + + expand = prop.find(xmlutils.make_clark("C:expand")) + if expand is not None: + start = expand.get('start') + end = expand.get('end') + + if (start is None) or (end is None): + return client.FORBIDDEN, \ + xmlutils.webdav_error("C:expand") + + start = datetime.datetime.strptime( + start, '%Y%m%dT%H%M%SZ' + ).replace(tzinfo=datetime.timezone.utc) + end = datetime.datetime.strptime( + end, '%Y%m%dT%H%M%SZ' + ).replace(tzinfo=datetime.timezone.utc) + + expanded_elements = _expand( + element, copy.copy(item), start, end) + found_props.extend(expanded_elements) + else: + found_props.append(element) else: not_found_props.append(element) @@ -164,6 +189,35 @@ def xml_report(base_prefix: str, path: str, xml_request: Optional[ET.Element], return client.MULTI_STATUS, multistatus +def _expand( + element: ET.Element, + item: radicale_item.Item, + start: datetime.datetime, + end: datetime.datetime, +) -> List[ET.Element]: + expanded = [element] + + for component in item.vobject_item.components(): + if component.name != 'VEVENT': + continue + + if hasattr(component, "rrule"): + rulleset = component.getrruleset() + instances = rulleset.between(start, end) + + for instance in instances: + try: + delattr(item.vobject_item.vevent, 'recurrence-id') + except AttributeError: + pass + + item.vobject_item.vevent.add('RECURRENCE-ID').value = instance + element.text = item.vobject_item.serialize() + expanded.append(element) + + return expanded + + def xml_item_response(base_prefix: str, href: str, found_props: Sequence[ET.Element] = (), not_found_props: Sequence[ET.Element] = (), From a07813ecc96f326b0c7217c223f744a4db9f2a75 Mon Sep 17 00:00:00 2001 From: Georgiy Date: Thu, 30 Mar 2023 19:43:15 +0300 Subject: [PATCH 2/7] fix variables naming, fix recurrence element duplication --- radicale/app/report.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/radicale/app/report.py b/radicale/app/report.py index d5c94f8c..e9cd57d9 100644 --- a/radicale/app/report.py +++ b/radicale/app/report.py @@ -203,15 +203,16 @@ def _expand( if hasattr(component, "rrule"): rulleset = component.getrruleset() - instances = rulleset.between(start, end) + recurrences = rulleset.between(start, end) - for instance in instances: + expanded = [] + for recurrence_dt in recurrences: try: delattr(item.vobject_item.vevent, 'recurrence-id') except AttributeError: pass - item.vobject_item.vevent.add('RECURRENCE-ID').value = instance + item.vobject_item.vevent.add('RECURRENCE-ID').value = recurrence_dt element.text = item.vobject_item.serialize() expanded.append(element) From 72103c30c2717cf5710749251586403871429af5 Mon Sep 17 00:00:00 2001 From: Georgiy Date: Thu, 30 Mar 2023 23:13:00 +0300 Subject: [PATCH 3/7] recurring events brought to rfc4791 --- radicale/app/report.py | 77 ++++++++++++++++++++++++++++++++++-------- 1 file changed, 62 insertions(+), 15 deletions(-) diff --git a/radicale/app/report.py b/radicale/app/report.py index e9cd57d9..415bf0de 100644 --- a/radicale/app/report.py +++ b/radicale/app/report.py @@ -23,6 +23,7 @@ import posixpath import socket import copy import xml.etree.ElementTree as ET +from vobject.base import ContentLine from http import client from typing import ( Callable, Iterable, Iterator, @@ -197,28 +198,74 @@ def _expand( ) -> List[ET.Element]: expanded = [element] - for component in item.vobject_item.components(): - if component.name != 'VEVENT': - continue + if hasattr(item.vobject_item.vevent, "rrule"): + rulleset = item.vobject_item.vevent.getrruleset() + recurrences = rulleset.between(start, end) + recurring_item = _make_vobject_recurring_item(item) - if hasattr(component, "rrule"): - rulleset = component.getrruleset() - recurrences = rulleset.between(start, end) + expanded = [] + for recurrence_dt in recurrences: + try: + delattr(recurring_item.vobject_item.vevent, 'recurrence-id') + except AttributeError: + pass - expanded = [] - for recurrence_dt in recurrences: - try: - delattr(item.vobject_item.vevent, 'recurrence-id') - except AttributeError: - pass + recurrence_utc = recurrence_dt.astimezone(datetime.timezone.utc) - item.vobject_item.vevent.add('RECURRENCE-ID').value = recurrence_dt - element.text = item.vobject_item.serialize() - expanded.append(element) + recurring_item.vobject_item.vevent.recurrence_id = ContentLine( + name='RECURRENCE-ID', + value=recurrence_utc.strftime('%Y%m%dT%H%M%SZ'), params={} + ) + + element = copy.copy(element) + element.text = recurring_item.vobject_item.serialize() + expanded.append(element) return expanded +def _make_vobject_recurring_item( + item: radicale_item.Item +) -> radicale_item.Item: + # https://www.rfc-editor.org/rfc/rfc4791#section-9.6.5 + # The returned calendar components MUST NOT use recurrence + # properties (i.e., EXDATE, EXRULE, RDATE, and RRULE) and MUST NOT + # have reference to or include VTIMEZONE components. Date and local + # time with reference to time zone information MUST be converted + # into date with UTC time. + + item = copy.copy(item) + vevent = item.vobject_item.vevent + + start_utc = vevent.dtstart.value.astimezone(datetime.timezone.utc) + end_utc = vevent.dtend.value.astimezone(datetime.timezone.utc) + + vevent.dtstart = ContentLine( + name='DTSTART', + value=start_utc.strftime('%Y%m%dT%H%M%SZ'), params={}) + vevent.dtend = ContentLine( + name='DTEND', + value=end_utc.strftime('%Y%m%dT%H%M%SZ'), params={}) + + timezones_to_remove = [] + for component in item.vobject_item.components(): + if component.name == 'VTIMEZONE': + timezones_to_remove.append(component) + + for timezone in timezones_to_remove: + item.vobject_item.remove(timezone) + + try: + delattr(item.vobject_item.vevent, 'rrule') + delattr(item.vobject_item.vevent, 'exdate') + delattr(item.vobject_item.vevent, 'exrule') + delattr(item.vobject_item.vevent, 'rdate') + except AttributeError: + pass + + return item + + def xml_item_response(base_prefix: str, href: str, found_props: Sequence[ET.Element] = (), not_found_props: Sequence[ET.Element] = (), From 513e04e6363814607fa02e5182cc3b3e709e0182 Mon Sep 17 00:00:00 2001 From: Georgiy Date: Fri, 31 Mar 2023 12:00:59 +0300 Subject: [PATCH 4/7] all expanded components has the same view --- radicale/app/report.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/radicale/app/report.py b/radicale/app/report.py index 415bf0de..4420f0e0 100644 --- a/radicale/app/report.py +++ b/radicale/app/report.py @@ -196,35 +196,36 @@ def _expand( start: datetime.datetime, end: datetime.datetime, ) -> List[ET.Element]: + expanded_item = _make_vobject_expanded_item(item) + element.text = expanded_item.vobject_item.serialize() expanded = [element] if hasattr(item.vobject_item.vevent, "rrule"): rulleset = item.vobject_item.vevent.getrruleset() recurrences = rulleset.between(start, end) - recurring_item = _make_vobject_recurring_item(item) expanded = [] for recurrence_dt in recurrences: try: - delattr(recurring_item.vobject_item.vevent, 'recurrence-id') + delattr(expanded_item.vobject_item.vevent, 'recurrence-id') except AttributeError: pass recurrence_utc = recurrence_dt.astimezone(datetime.timezone.utc) - recurring_item.vobject_item.vevent.recurrence_id = ContentLine( + expanded_item.vobject_item.vevent.recurrence_id = ContentLine( name='RECURRENCE-ID', value=recurrence_utc.strftime('%Y%m%dT%H%M%SZ'), params={} ) element = copy.copy(element) - element.text = recurring_item.vobject_item.serialize() + element.text = expanded_item.vobject_item.serialize() expanded.append(element) return expanded -def _make_vobject_recurring_item( +def _make_vobject_expanded_item( item: radicale_item.Item ) -> radicale_item.Item: # https://www.rfc-editor.org/rfc/rfc4791#section-9.6.5 From 513415d2015d28754bcea880bb841d7777013c4e Mon Sep 17 00:00:00 2001 From: Georgiy Date: Mon, 1 Apr 2024 19:27:59 +0300 Subject: [PATCH 5/7] start creating test for rrule expand property, fix expand processing --- radicale/app/report.py | 54 +++++++++++++-------- radicale/tests/static/event_daily_rrule.ics | 28 +++++++++++ radicale/tests/test_base.py | 43 ++++++++++++++++ 3 files changed, 105 insertions(+), 20 deletions(-) create mode 100644 radicale/tests/static/event_daily_rrule.ics diff --git a/radicale/app/report.py b/radicale/app/report.py index 4420f0e0..08e1fed3 100644 --- a/radicale/app/report.py +++ b/radicale/app/report.py @@ -172,9 +172,9 @@ def xml_report(base_prefix: str, path: str, xml_request: Optional[ET.Element], end, '%Y%m%dT%H%M%SZ' ).replace(tzinfo=datetime.timezone.utc) - expanded_elements = _expand( + expanded_element = _expand( element, copy.copy(item), start, end) - found_props.extend(expanded_elements) + found_props.append(expanded_element) else: found_props.append(element) else: @@ -195,34 +195,46 @@ def _expand( item: radicale_item.Item, start: datetime.datetime, end: datetime.datetime, -) -> List[ET.Element]: +) -> ET.Element: + rruleset = None + if hasattr(item.vobject_item.vevent, 'rrule'): + rruleset = item.vobject_item.vevent.getrruleset() + expanded_item = _make_vobject_expanded_item(item) - element.text = expanded_item.vobject_item.serialize() - expanded = [element] - if hasattr(item.vobject_item.vevent, "rrule"): - rulleset = item.vobject_item.vevent.getrruleset() - recurrences = rulleset.between(start, end) + if rruleset: + recurrences = rruleset.between(start, end) - expanded = [] + expanded = None for recurrence_dt in recurrences: + expanded_item_ = copy.copy(expanded_item) + try: - delattr(expanded_item.vobject_item.vevent, 'recurrence-id') + delattr(expanded_item_.vobject_item.vevent, 'recurrence-id') except AttributeError: pass recurrence_utc = recurrence_dt.astimezone(datetime.timezone.utc) - expanded_item.vobject_item.vevent.recurrence_id = ContentLine( + vevent = copy.deepcopy(expanded_item_.vobject_item.vevent) + recurrence_id = ContentLine( name='RECURRENCE-ID', value=recurrence_utc.strftime('%Y%m%dT%H%M%SZ'), params={} ) + vevent.add(recurrence_id) - element = copy.copy(element) - element.text = expanded_item.vobject_item.serialize() - expanded.append(element) + if expanded is None: + expanded_item_.vobject_item.vevent.add(recurrence_id) + expanded_item_.vobject_item.remove(expanded_item_.vobject_item.vevent) + expanded = expanded_item_ + else: + expanded.vobject_item.add(vevent) - return expanded + element.text = expanded.vobject_item.serialize() + else: + element.text = expanded_item.vobject_item.serialize() + + return element def _make_vobject_expanded_item( @@ -239,14 +251,16 @@ def _make_vobject_expanded_item( vevent = item.vobject_item.vevent start_utc = vevent.dtstart.value.astimezone(datetime.timezone.utc) - end_utc = vevent.dtend.value.astimezone(datetime.timezone.utc) - vevent.dtstart = ContentLine( name='DTSTART', value=start_utc.strftime('%Y%m%dT%H%M%SZ'), params={}) - vevent.dtend = ContentLine( - name='DTEND', - value=end_utc.strftime('%Y%m%dT%H%M%SZ'), params={}) + + dt_end = getattr(vevent, 'dtend', None) + if dt_end is not None: + end_utc = dt_end.value.astimezone(datetime.timezone.utc) + vevent.dtend = ContentLine( + name='DTEND', + value=end_utc.strftime('%Y%m%dT%H%M%SZ'), params={}) timezones_to_remove = [] for component in item.vobject_item.components(): diff --git a/radicale/tests/static/event_daily_rrule.ics b/radicale/tests/static/event_daily_rrule.ics new file mode 100644 index 00000000..362a18e4 --- /dev/null +++ b/radicale/tests/static/event_daily_rrule.ics @@ -0,0 +1,28 @@ +BEGIN:VCALENDAR +VERSION:2.0 +BEGIN:VTIMEZONE +LAST-MODIFIED:20040110T032845Z +TZID:US/Eastern +BEGIN:DAYLIGHT +DTSTART:20000404T020000 +RRULE:FREQ=YEARLY;BYDAY=1SU;BYMONTH=4 +TZNAME:EDT +TZOFFSETFROM:-0500 +TZOFFSETTO:-0400 +END:DAYLIGHT +BEGIN:STANDARD +DTSTART:20001026T020000 +RRULE:FREQ=YEARLY;BYDAY=-1SU;BYMONTH=10 +TZNAME:EST +TZOFFSETFROM:-0400 +TZOFFSETTO:-0500 +END:STANDARD +END:VTIMEZONE +BEGIN:VEVENT +DTSTART;TZID=US/Eastern:20060102T120000 +DURATION:PT1H +RRULE:FREQ=DAILY;COUNT=5 +SUMMARY:Recurring event +UID:event_daily_rrule +END:VEVENT +END:VCALENDAR diff --git a/radicale/tests/test_base.py b/radicale/tests/test_base.py index a0d3d534..6c1ac1b3 100644 --- a/radicale/tests/test_base.py +++ b/radicale/tests/test_base.py @@ -1525,6 +1525,49 @@ permissions: RrWw""") calendar_path, "http://radicale.org/ns/sync/INVALID") assert not sync_token + def test_report_with_expand_property(self) -> None: + self.put("/calendar.ics/", get_file_content("event_daily_rrule.ics")) + + req_body = \ + """ + + + + + + + + + + + + + + + """ + + # status, _, answer = self.request("REPORT", "/calendar.ics/", req_body, check=207) + # print(status, answer) + + _, responses = self.report("/calendar.ics/", req_body) + assert len(responses) == 1 + response = responses['/calendar.ics/event_daily_rrule.ics'] + status, element = list(response.values())[0] + assert status == 200 + + print("resp", status, element, flush=True) + + uids = [] + for line in element.text.split("\n"): + print("line", line, line.startswith("UID:")) + if line.startswith("UID:"): + uid = line[len("UID:"):] + assert uid == "event_daily_rrule" + uids.append(uids) + + assert len(uids) == 3 + assert False + def test_propfind_sync_token(self) -> None: """Retrieve the sync-token with a propfind request""" calendar_path = "/calendar.ics/" From d1da63569b3bfd238d47ed5746250b5fa1512d35 Mon Sep 17 00:00:00 2001 From: Georgiy Date: Mon, 1 Apr 2024 19:52:16 +0300 Subject: [PATCH 6/7] Fix setting recurrence-id for expanded items. test for report with expand finished --- radicale/app/report.py | 21 +++++-------- radicale/tests/test_base.py | 60 ++++++++++++++++++++++++++++++------- 2 files changed, 56 insertions(+), 25 deletions(-) diff --git a/radicale/app/report.py b/radicale/app/report.py index 08e1fed3..dc9638b5 100644 --- a/radicale/app/report.py +++ b/radicale/app/report.py @@ -207,30 +207,23 @@ def _expand( expanded = None for recurrence_dt in recurrences: - expanded_item_ = copy.copy(expanded_item) - - try: - delattr(expanded_item_.vobject_item.vevent, 'recurrence-id') - except AttributeError: - pass + vobject_item = copy.copy(expanded_item.vobject_item) recurrence_utc = recurrence_dt.astimezone(datetime.timezone.utc) - vevent = copy.deepcopy(expanded_item_.vobject_item.vevent) - recurrence_id = ContentLine( + vevent = copy.deepcopy(vobject_item.vevent) + vevent.recurrence_id = ContentLine( name='RECURRENCE-ID', value=recurrence_utc.strftime('%Y%m%dT%H%M%SZ'), params={} ) - vevent.add(recurrence_id) if expanded is None: - expanded_item_.vobject_item.vevent.add(recurrence_id) - expanded_item_.vobject_item.remove(expanded_item_.vobject_item.vevent) - expanded = expanded_item_ + vobject_item.vevent = vevent + expanded = vobject_item else: - expanded.vobject_item.add(vevent) + expanded.add(vevent) - element.text = expanded.vobject_item.serialize() + element.text = expanded.serialize() else: element.text = expanded_item.vobject_item.serialize() diff --git a/radicale/tests/test_base.py b/radicale/tests/test_base.py index 6c1ac1b3..0743dd6c 100644 --- a/radicale/tests/test_base.py +++ b/radicale/tests/test_base.py @@ -1527,8 +1527,44 @@ permissions: RrWw""") def test_report_with_expand_property(self) -> None: self.put("/calendar.ics/", get_file_content("event_daily_rrule.ics")) + req_body_without_expand = \ + """ + + + + + + + + + + + + + + """ + _, responses = self.report("/calendar.ics/", req_body_without_expand) + assert len(responses) == 1 - req_body = \ + response = responses['/calendar.ics/event_daily_rrule.ics'] + status, element = list(response.values())[0] + + assert status == 200 + + assert "RRULE" in element.text + assert "BEGIN:VTIMEZONE" in element.text + assert "RECURRENCE-ID" not in element.text + + uids = [] + for line in element.text.split("\n"): + if line.startswith("UID:"): + uid = line[len("UID:"):] + assert uid == "event_daily_rrule" + uids.append(uids) + + assert len(uids) == 1 + + req_body_with_expand = \ """ @@ -1546,27 +1582,29 @@ permissions: RrWw""") """ - # status, _, answer = self.request("REPORT", "/calendar.ics/", req_body, check=207) - # print(status, answer) + _, responses = self.report("/calendar.ics/", req_body_with_expand) - _, responses = self.report("/calendar.ics/", req_body) assert len(responses) == 1 + response = responses['/calendar.ics/event_daily_rrule.ics'] status, element = list(response.values())[0] - assert status == 200 - print("resp", status, element, flush=True) + assert status == 200 + assert "RRULE" not in element.text + assert "BEGIN:VTIMEZONE" not in element.text uids = [] + recurrence_ids = [] for line in element.text.split("\n"): - print("line", line, line.startswith("UID:")) if line.startswith("UID:"): - uid = line[len("UID:"):] - assert uid == "event_daily_rrule" + assert line == "UID:event_daily_rrule" uids.append(uids) - assert len(uids) == 3 - assert False + if line.startswith("RECURRENCE-ID:"): + recurrence_ids.append(line) + + assert len(uids) == 2 + assert len(set(recurrence_ids)) == 2 def test_propfind_sync_token(self) -> None: """Retrieve the sync-token with a propfind request""" From 4e4af2aca57ba294f2126eb57b28c17589497222 Mon Sep 17 00:00:00 2001 From: Georgiy Date: Fri, 5 Apr 2024 11:35:53 +0300 Subject: [PATCH 7/7] pep8 fix --- radicale/app/report.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/radicale/app/report.py b/radicale/app/report.py index dc9638b5..0c869107 100644 --- a/radicale/app/report.py +++ b/radicale/app/report.py @@ -27,7 +27,7 @@ from vobject.base import ContentLine from http import client from typing import ( Callable, Iterable, Iterator, - Optional, Sequence, Tuple, List, + Optional, Sequence, Tuple, ) from urllib.parse import unquote, urlparse