Merge branch 'master' into react0r

.github/workflows/test.yml

@@ -6,7 +6,7 @@ jobs:
     strategy:
       matrix:
         os: [ubuntu-latest, macos-latest, windows-latest]
-        python-version: ['3.8', '3.9', '3.10', '3.11', '3.12', pypy-3.8, pypy-3.9]
+        python-version: ['3.8', '3.9', '3.10', '3.11', '3.12.3', '3.13.0-beta.4', pypy-3.8, pypy-3.9]
         exclude:
           - os: windows-latest
             python-version: pypy-3.8
@@ -21,7 +21,7 @@ jobs:
       - name: Install Test dependencies
         run: pip install tox
       - name: Test
-        run: tox
+        run: tox -e py
       - name: Install Coveralls
         if: github.event_name == 'push'
         run: pip install coveralls
@@ -46,3 +46,15 @@ jobs:
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         run: coveralls --service=github --finish
+
+  lint:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: '3.12'
+      - name: Install tox
+        run: pip install tox
+      - name: Lint
+        run: tox -e flake8,mypy,isort

CHANGELOG.md

@@ -4,6 +4,8 @@
 * Fix: Using icalendar's tzinfo on created datetime to fix issue with icalendar
 * Enhancement: Added free-busy report
 * Enhancement: Added 'max_freebusy_occurrences` setting to avoid potential DOS on reports
+* Enhancement: remove unexpected control codes from uploaded items
+* Drop: remove unused requirement "typeguard"
 * Improve: Refactored some date parsing code
 
 ## 3.2.2

DOCUMENTATION.md

@@ -350,16 +350,13 @@ location /radicale/ { # The trailing / is important!
 }
 ```
 
-Example **Caddy** configuration with basicauth from Caddy:
+Example **Caddy** configuration:
 
-```Caddy
+```
-handle_path /radicale* {
+handle_path /radicale/* {
-    basicauth {
+    uri strip_prefix /radicale
-        user hash
-    }
     reverse_proxy localhost:5232 {
-        header_up +X-Script-Name "/radicale"
+        header_up X-Script-Name /radicale
-        header_up +X-remote-user "{http.auth.user.id}"
     }
 }
 ```
@@ -440,6 +437,21 @@ location /radicale/ {
 }
 ```
 
+Example **Caddy** configuration:
+
+```
+handle_path /radicale/* {
+    uri strip_prefix /radicale
+    basicauth {
+        USER HASH
+    }
+    reverse_proxy localhost:5232 {
+        header_up X-Script-Name /radicale
+        header_up X-remote-user {http.auth.user.id}
+    }
+}
+```
+
 Example **Apache** configuration:
 
 ```apache
@@ -795,6 +807,12 @@ providers like ldap, kerberos
 
 Default: `False`
 
+##### strip_domain
+
+Strip domain from username
+
+Default: `False`
+
 #### rights
 
 ##### type
@@ -865,7 +883,7 @@ Delete sync-token that are older than the specified time. (seconds)
 
 Default: `2592000`
 
-#### skip_broken_item
+##### skip_broken_item
 
 Skip broken item instead of triggering an exception
 

config

@@ -73,6 +73,8 @@
 # Convert username to lowercase, must be true for case-insensitive auth providers
 #lc_username = False
 
+# Strip domain name from username
+#strip_domain = False
 
 [rights]
 

radicale/__init__.py

@@ -61,7 +61,7 @@ def _get_application_instance(config_path: str, wsgi_errors: types.ErrorStream
                     if not miss and source != "default config":
                         default_config_active = False
                 if default_config_active:
-                    logger.warn("%s", "No config file found/readable - only default config is active")
+                    logger.warning("%s", "No config file found/readable - only default config is active")
                 _application_instance = Application(configuration)
     if _application_config_path != config_path:
         raise ValueError("RADICALE_CONFIG must not change: %r != %r" %

radicale/__main__.py

@@ -175,7 +175,7 @@ def run() -> None:
             default_config_active = False
 
     if default_config_active:
-        logger.warn("%s", "No config file found/readable - only default config is active")
+        logger.warning("%s", "No config file found/readable - only default config is active")
 
     if args_ns.verify_storage:
         logger.info("Verifying storage")
@@ -183,7 +183,7 @@ def run() -> None:
             storage_ = storage.load(configuration)
             with storage_.acquire_lock("r"):
                 if not storage_.verify():
-                    logger.critical("Storage verifcation failed")
+                    logger.critical("Storage verification failed")
                     sys.exit(1)
         except Exception as e:
             logger.critical("An exception occurred during storage "

radicale/app/__init__.py

@@ -232,7 +232,7 @@ class Application(ApplicationPartDelete, ApplicationPartHead,
                 path.rstrip("/").endswith("/.well-known/carddav")):
             return response(*httputils.redirect(
                 base_prefix + "/", client.MOVED_PERMANENTLY))
-        # Return NOT FOUND for all other paths containing ".well-knwon"
+        # Return NOT FOUND for all other paths containing ".well-known"
         if path.endswith("/.well-known") or "/.well-known/" in path:
             return response(*httputils.NOT_FOUND)
 

radicale/app/propfind.py

@@ -322,13 +322,13 @@ def xml_propfind_response(
 
         responses[404 if is404 else 200].append(element)
 
-    for status_code, childs in responses.items():
+    for status_code, children in responses.items():
-        if not childs:
+        if not children:
             continue
         propstat = ET.Element(xmlutils.make_clark("D:propstat"))
         response.append(propstat)
         prop = ET.Element(xmlutils.make_clark("D:prop"))
-        prop.extend(childs)
+        prop.extend(children)
         propstat.append(prop)
         status = ET.Element(xmlutils.make_clark("D:status"))
         status.text = xmlutils.make_response(status_code)

radicale/app/report.py

@@ -363,7 +363,7 @@ def _make_vobject_expanded_item(
     if hasattr(item.vobject_item.vevent, 'rrule'):
         rruleset = vevent.getrruleset()
 
-    # There is something strage behavour during serialization native datetime, so converting manualy
+    # There is something strange behaviour during serialization native datetime, so converting manually
     vevent.dtstart.value = vevent.dtstart.value.strftime(dt_format)
     if dt_end is not None:
         vevent.dtend.value = vevent.dtend.value.strftime(dt_format)

radicale/auth/__init__.py

@@ -52,6 +52,7 @@ def load(configuration: "config.Configuration") -> "BaseAuth":
 class BaseAuth:
 
     _lc_username: bool
+    _strip_domain: bool
 
     def __init__(self, configuration: "config.Configuration") -> None:
         """Initialize BaseAuth.
@@ -63,6 +64,7 @@ class BaseAuth:
         """
         self.configuration = configuration
         self._lc_username = configuration.get("auth", "lc_username")
+        self._strip_domain = configuration.get("auth", "strip_domain")
 
     def get_external_login(self, environ: types.WSGIEnviron) -> Union[
             Tuple[()], Tuple[str, str]]:
@@ -91,4 +93,8 @@ class BaseAuth:
         raise NotImplementedError
 
     def login(self, login: str, password: str) -> str:
-        return self._login(login, password).lower() if self._lc_username else self._login(login, password)
+        if self._lc_username:
+            login = login.lower()
+        if self._strip_domain:
+            login = login.split('@')[0]
+        return self._login(login, password)

radicale/auth/htpasswd.py

@@ -36,7 +36,7 @@ pointed to by the ``htpasswd_filename`` configuration value while assuming
 the password encryption method specified via the ``htpasswd_encryption``
 configuration value.
 
-The following htpasswd password encrpytion methods are supported by Radicale
+The following htpasswd password encryption methods are supported by Radicale
 out-of-the-box:
     - plain-text (created by htpasswd -p ...) -- INSECURE
     - MD5-APR1   (htpasswd -m ...) -- htpasswd's default method, INSECURE

radicale/config.py

@@ -191,6 +191,10 @@ DEFAULT_CONFIG_SCHEMA: types.CONFIG_SCHEMA = OrderedDict([
             "value": "1",
             "help": "incorrect authentication delay",
             "type": positive_float}),
+        ("strip_domain", {
+            "value": "False",
+            "help": "strip domain from username",
+            "type": bool}),
         ("lc_username", {
             "value": "False",
             "help": "convert username to lowercase, must be true for case-insensitive auth providers",

radicale/hook/__init__.py

@@ -3,12 +3,21 @@ from enum import Enum
 from typing import Sequence
 
 from radicale import pathutils, utils
+from radicale.log import logger
 
 INTERNAL_TYPES: Sequence[str] = ("none", "rabbitmq")
 
 
 def load(configuration):
     """Load the storage module chosen in configuration."""
+    try:
+        return utils.load_plugin(
+            INTERNAL_TYPES, "hook", "Hook", BaseHook, configuration)
+    except Exception as e:
+        logger.warn(e)
+        logger.warn("Hook \"%s\" failed to load, falling back to \"none\"." % configuration.get("hook", "type"))
+        configuration = configuration.copy()
+        configuration.update({"hook": {"type": "none"}}, "hook", privileged=True)
         return utils.load_plugin(
             INTERNAL_TYPES, "hook", "Hook", BaseHook, configuration)
 

radicale/item/__init__.py

@@ -49,6 +49,12 @@ def read_components(s: str) -> List[vobject.base.Component]:
     s = re.sub(r"^(PHOTO(?:;[^:\r\n]*)?;ENCODING=b(?:;[^:\r\n]*)?:)"
                r"data:[^;,\r\n]*;base64,", r"\1", s,
                flags=re.MULTILINE | re.IGNORECASE)
+    # Workaround for bug with malformed ICS files containing control codes
+    # Filter out all control codes except those we expect to find:
+    #  * 0x09 Horizontal Tab
+    #  * 0x0A Line Feed
+    #  * 0x0D Carriage Return
+    s = re.sub(r'[\x00-\x08\x0B\x0C\x0E-\x1F]', '', s)
     return list(vobject.readComponents(s, allowQP=True))
 
 
@@ -298,7 +304,7 @@ def find_time_range(vobject_item: vobject.base.Component, tag: str
     Returns a tuple (``start``, ``end``) where ``start`` and ``end`` are
     POSIX timestamps.
 
-    This is intened to be used for matching against simplified prefilters.
+    This is intended to be used for matching against simplified prefilters.
 
     """
     if not tag:

radicale/item/filter.py

@@ -241,7 +241,7 @@ def visit_time_ranges(vobject_item: vobject.base.Component, child_name: str,
 
     """
 
-    # HACK: According to rfc5545-3.8.4.4 an recurrance that is resheduled
+    # HACK: According to rfc5545-3.8.4.4 a recurrence that is rescheduled
     # with Recurrence ID affects the recurrence itself and all following
     # recurrences too. This is not respected and client don't seem to bother
     # either.

radicale/rights/from_file.py

@@ -22,7 +22,7 @@ config (section "rights", key "file").
 The login is matched against the "user" key, and the collection path
 is matched against the "collection" key. In the "collection" regex you can use
 `{user}` and get groups from the "user" regex with `{0}`, `{1}`, etc.
-In consequence of the parameter subsitution you have to write `{{` and `}}`
+In consequence of the parameter substitution you have to write `{{` and `}}`
 if you want to use regular curly braces in the "user" and "collection" regexes.
 
 For example, for the "user" key, ".+" means "authenticated user" and ".*"

radicale/server.py

@@ -291,7 +291,7 @@ def serve(configuration: config.Configuration,
             try:
                 getaddrinfo = socket.getaddrinfo(address_port[0], address_port[1], 0, socket.SOCK_STREAM, socket.IPPROTO_TCP)
             except OSError as e:
-                logger.warn("cannot retrieve IPv4 or IPv6 address of '%s': %s" % (format_address(address_port), e))
+                logger.warning("cannot retrieve IPv4 or IPv6 address of '%s': %s" % (format_address(address_port), e))
                 continue
             logger.debug("getaddrinfo of '%s': %s" % (format_address(address_port), getaddrinfo))
             for (address_family, socket_kind, socket_proto, socket_flags, socket_address) in getaddrinfo:
@@ -299,7 +299,7 @@ def serve(configuration: config.Configuration,
                 try:
                     server = server_class(configuration, address_family, (socket_address[0], socket_address[1]), RequestHandler)
                 except OSError as e:
-                    logger.warn("cannot create server socket on '%s': %s" % (format_address(socket_address), e))
+                    logger.warning("cannot create server socket on '%s': %s" % (format_address(socket_address), e))
                     continue
                 servers[server.socket] = server
                 server.set_app(application)

radicale/storage/multifilesystem/get.py

@@ -84,7 +84,7 @@ class CollectionPartGet(CollectionPartCache, CollectionPartLock,
         cache_content = self._load_item_cache(href, cache_hash)
         if cache_content is None:
             with self._acquire_cache_lock("item"):
-                # Lock the item cache to prevent multpile processes from
+                # Lock the item cache to prevent multiple processes from
                 # generating the same data in parallel.
                 # This improves the performance for multiple requests.
                 if self._storage._lock.locked == "r":
@@ -127,7 +127,7 @@ class CollectionPartGet(CollectionPartCache, CollectionPartLock,
 
     def get_multi(self, hrefs: Iterable[str]
                   ) -> Iterator[Tuple[str, Optional[radicale_item.Item]]]:
-        # It's faster to check for file name collissions here, because
+        # It's faster to check for file name collisions here, because
         # we only need to call os.listdir once.
         files = None
         for href in hrefs:
@@ -146,7 +146,7 @@ class CollectionPartGet(CollectionPartCache, CollectionPartLock,
 
     def get_all(self) -> Iterator[radicale_item.Item]:
         for href in self._list():
-            # We don't need to check for collissions, because the file names
+            # We don't need to check for collisions, because the file names
             # are from os.listdir.
             item = self._get(href, verify_href=False)
             if item is not None:

radicale/tests/__init__.py

@@ -112,7 +112,7 @@ class BaseTest:
         for response in xml.findall(xmlutils.make_clark("D:response")):
             href = response.find(xmlutils.make_clark("D:href"))
             assert href.text not in path_responses
-            prop_respones: Dict[str, Tuple[int, ET.Element]] = {}
+            prop_responses: Dict[str, Tuple[int, ET.Element]] = {}
             for propstat in response.findall(
                     xmlutils.make_clark("D:propstat")):
                 status = propstat.find(xmlutils.make_clark("D:status"))
@@ -121,16 +121,16 @@ class BaseTest:
                 for element in propstat.findall(
                         "./%s/*" % xmlutils.make_clark("D:prop")):
                     human_tag = xmlutils.make_human_tag(element.tag)
-                    assert human_tag not in prop_respones
+                    assert human_tag not in prop_responses
-                    prop_respones[human_tag] = (status_code, element)
+                    prop_responses[human_tag] = (status_code, element)
             status = response.find(xmlutils.make_clark("D:status"))
             if status is not None:
-                assert not prop_respones
+                assert not prop_responses
                 assert status.text.startswith("HTTP/1.1 ")
                 status_code = int(status.text.split(" ")[1])
                 path_responses[href.text] = status_code
             else:
-                path_responses[href.text] = prop_respones
+                path_responses[href.text] = prop_responses
         return path_responses
 
     @staticmethod

radicale/tests/test_auth.py

@@ -115,6 +115,16 @@ class TestBaseAuthRequests(BaseTest):
     def test_htpasswd_comment(self) -> None:
         self._test_htpasswd("plain", "#comment\n #comment\n \ntmp:bepo\n\n")
 
+    def test_htpasswd_lc_username(self) -> None:
+        self.configure({"auth": {"lc_username": "True"}})
+        self._test_htpasswd("plain", "tmp:bepo", (
+            ("tmp", "bepo", True), ("TMP", "bepo", True), ("tmp1", "bepo", False)))
+
+    def test_htpasswd_strip_domain(self) -> None:
+        self.configure({"auth": {"strip_domain": "True"}})
+        self._test_htpasswd("plain", "tmp:bepo", (
+            ("tmp", "bepo", True), ("tmp@domain.example", "bepo", True), ("tmp1", "bepo", False)))
+
     def test_remote_user(self) -> None:
         self.configure({"auth": {"type": "remote_user"}})
         _, responses = self.propfind("/", """\

radicale/tests/test_base.py

@@ -360,7 +360,7 @@ permissions: RrWw""")
         self.get(path1, check=404)
         self.get(path2)
 
-    def test_move_between_colections(self) -> None:
+    def test_move_between_collections(self) -> None:
         """Move a item."""
         self.mkcalendar("/calendar1.ics/")
         self.mkcalendar("/calendar2.ics/")
@@ -373,7 +373,7 @@ permissions: RrWw""")
         self.get(path1, check=404)
         self.get(path2)
 
-    def test_move_between_colections_duplicate_uid(self) -> None:
+    def test_move_between_collections_duplicate_uid(self) -> None:
         """Move a item to a collection which already contains the UID."""
         self.mkcalendar("/calendar1.ics/")
         self.mkcalendar("/calendar2.ics/")
@@ -389,7 +389,7 @@ permissions: RrWw""")
         assert xml.tag == xmlutils.make_clark("D:error")
         assert xml.find(xmlutils.make_clark("C:no-uid-conflict")) is not None
 
-    def test_move_between_colections_overwrite(self) -> None:
+    def test_move_between_collections_overwrite(self) -> None:
         """Move a item to a collection which already contains the item."""
         self.mkcalendar("/calendar1.ics/")
         self.mkcalendar("/calendar2.ics/")
@@ -403,8 +403,8 @@ permissions: RrWw""")
         self.request("MOVE", path1, check=204, HTTP_OVERWRITE="T",
                      HTTP_DESTINATION="http://127.0.0.1/"+path2)
 
-    def test_move_between_colections_overwrite_uid_conflict(self) -> None:
+    def test_move_between_collections_overwrite_uid_conflict(self) -> None:
-        """Move a item to a collection which already contains the item with
+        """Move an item to a collection which already contains the item with
            a different UID."""
         self.mkcalendar("/calendar1.ics/")
         self.mkcalendar("/calendar2.ics/")

setup.cfg

@@ -1,26 +1,31 @@
 [tool:pytest]
-addopts = --typeguard-packages=radicale
 
 [tox:tox]
+min_version = 4.0
+envlist = py, flake8, isort, mypy
 
 [testenv]
-extras = test
+extras =
+    test
 deps =
-    flake8
+    pytest
-    isort
-    # mypy installation fails with pypy<3.9
-    mypy; implementation_name!='pypy' or python_version>='3.9'
-    types-setuptools
     pytest-cov
-commands =
+commands = pytest -r s --cov --cov-report=term --cov-report=xml .
-    flake8 .
+
-    isort --check --diff .
+[testenv:flake8]
-    # Run mypy if it's installed
+deps = flake8==7.1.0
-    python -c 'import importlib.util, subprocess, sys; \
+commands = flake8 .
-               importlib.util.find_spec("mypy") \
+skip_install = True
-                   and sys.exit(subprocess.run(["mypy", "."]).returncode) \
+
-                   or print("Skipped: mypy is not installed")'
+[testenv:isort]
-    pytest -r s --cov --cov-report=term --cov-report=xml .
+deps = isort==5.13.2
+commands = isort --check --diff .
+skip_install = True
+
+[testenv:mypy]
+deps = mypy==1.11.0
+commands = mypy .
+skip_install = True
 
 [tool:isort]
 known_standard_library = _dummy_thread,_thread,abc,aifc,argparse,array,ast,asynchat,asyncio,asyncore,atexit,audioop,base64,bdb,binascii,binhex,bisect,builtins,bz2,cProfile,calendar,cgi,cgitb,chunk,cmath,cmd,code,codecs,codeop,collections,colorsys,compileall,concurrent,configparser,contextlib,contextvars,copy,copyreg,crypt,csv,ctypes,curses,dataclasses,datetime,dbm,decimal,difflib,dis,distutils,doctest,dummy_threading,email,encodings,ensurepip,enum,errno,faulthandler,fcntl,filecmp,fileinput,fnmatch,formatter,fpectl,fractions,ftplib,functools,gc,getopt,getpass,gettext,glob,grp,gzip,hashlib,heapq,hmac,html,http,imaplib,imghdr,imp,importlib,inspect,io,ipaddress,itertools,json,keyword,lib2to3,linecache,locale,logging,lzma,macpath,mailbox,mailcap,marshal,math,mimetypes,mmap,modulefinder,msilib,msvcrt,multiprocessing,netrc,nis,nntplib,ntpath,numbers,operator,optparse,os,ossaudiodev,parser,pathlib,pdb,pickle,pickletools,pipes,pkgutil,platform,plistlib,poplib,posix,posixpath,pprint,profile,pstats,pty,pwd,py_compile,pyclbr,pydoc,queue,quopri,random,re,readline,reprlib,resource,rlcompleter,runpy,sched,secrets,select,selectors,shelve,shlex,shutil,signal,site,smtpd,smtplib,sndhdr,socket,socketserver,spwd,sqlite3,sre,sre_compile,sre_constants,sre_parse,ssl,stat,statistics,string,stringprep,struct,subprocess,sunau,symbol,symtable,sys,sysconfig,syslog,tabnanny,tarfile,telnetlib,tempfile,termios,test,textwrap,threading,time,timeit,tkinter,token,tokenize,trace,traceback,tracemalloc,tty,turtle,turtledemo,types,typing,unicodedata,unittest,urllib,uu,uuid,venv,warnings,wave,weakref,webbrowser,winreg,winsound,wsgiref,xdrlib,xml,xmlrpc,zipapp,zipfile,zipimport,zlib

setup.py

@@ -38,9 +38,9 @@ web_files = ["web/internal_data/css/icon.png",
 install_requires = ["defusedxml", "passlib", "vobject>=0.9.6",
                     "python-dateutil>=2.7.3",
                     "pika>=1.1.0",
-                    "setuptools; python_version<'3.9'"]
+                    ]
 bcrypt_requires = ["bcrypt"]
-test_requires = ["pytest>=7", "typeguard<4.3", "waitress", *bcrypt_requires]
+test_requires = ["pytest>=7", "waitress", *bcrypt_requires]
 
 setup(
     name="Radicale",
@@ -75,6 +75,7 @@ setup(
         "Programming Language :: Python :: 3.10",
         "Programming Language :: Python :: 3.11",
         "Programming Language :: Python :: 3.12",
+        "Programming Language :: Python :: 3.13",
         "Programming Language :: Python :: Implementation :: CPython",
         "Programming Language :: Python :: Implementation :: PyPy",
         "Topic :: Office/Business :: Groupware"])