diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 12f28633..32961e86 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -6,7 +6,7 @@ jobs: strategy: matrix: os: [ubuntu-latest, macos-latest, windows-latest] - python-version: ['3.8', '3.9', '3.10', '3.11', '3.12', pypy-3.8, pypy-3.9] + python-version: ['3.8', '3.9', '3.10', '3.11', '3.12.3', '3.13.0-beta.4', pypy-3.8, pypy-3.9] exclude: - os: windows-latest python-version: pypy-3.8 @@ -21,7 +21,7 @@ jobs: - name: Install Test dependencies run: pip install tox - name: Test - run: tox + run: tox -e py - name: Install Coveralls if: github.event_name == 'push' run: pip install coveralls @@ -46,3 +46,15 @@ jobs: env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: coveralls --service=github --finish + + lint: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + - name: Install tox + run: pip install tox + - name: Lint + run: tox -e flake8,mypy,isort diff --git a/CHANGELOG.md b/CHANGELOG.md index df11dd96..87d3fd2b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,8 @@ * Fix: Using icalendar's tzinfo on created datetime to fix issue with icalendar * Enhancement: Added free-busy report * Enhancement: Added 'max_freebusy_occurrences` setting to avoid potential DOS on reports +* Enhancement: remove unexpected control codes from uploaded items +* Drop: remove unused requirement "typeguard" * Improve: Refactored some date parsing code ## 3.2.2 diff --git a/DOCUMENTATION.md b/DOCUMENTATION.md index bace5789..c012b667 100644 --- a/DOCUMENTATION.md +++ b/DOCUMENTATION.md @@ -350,16 +350,13 @@ location /radicale/ { # The trailing / is important! } ``` -Example **Caddy** configuration with basicauth from Caddy: +Example **Caddy** configuration: -```Caddy -handle_path /radicale* { - basicauth { - user hash - } +``` +handle_path /radicale/* { + uri strip_prefix /radicale reverse_proxy localhost:5232 { - header_up +X-Script-Name "/radicale" - header_up +X-remote-user "{http.auth.user.id}" + header_up X-Script-Name /radicale } } ``` @@ -440,6 +437,21 @@ location /radicale/ { } ``` +Example **Caddy** configuration: + +``` +handle_path /radicale/* { + uri strip_prefix /radicale + basicauth { + USER HASH + } + reverse_proxy localhost:5232 { + header_up X-Script-Name /radicale + header_up X-remote-user {http.auth.user.id} + } +} +``` + Example **Apache** configuration: ```apache @@ -795,6 +807,12 @@ providers like ldap, kerberos Default: `False` +##### strip_domain + +Strip domain from username + +Default: `False` + #### rights ##### type @@ -865,7 +883,7 @@ Delete sync-token that are older than the specified time. (seconds) Default: `2592000` -#### skip_broken_item +##### skip_broken_item Skip broken item instead of triggering an exception diff --git a/config b/config index 114820ad..a9fe9da7 100644 --- a/config +++ b/config @@ -73,6 +73,8 @@ # Convert username to lowercase, must be true for case-insensitive auth providers #lc_username = False +# Strip domain name from username +#strip_domain = False [rights] diff --git a/radicale/__init__.py b/radicale/__init__.py index 2ce5d4b7..2554e5b2 100644 --- a/radicale/__init__.py +++ b/radicale/__init__.py @@ -61,7 +61,7 @@ def _get_application_instance(config_path: str, wsgi_errors: types.ErrorStream if not miss and source != "default config": default_config_active = False if default_config_active: - logger.warn("%s", "No config file found/readable - only default config is active") + logger.warning("%s", "No config file found/readable - only default config is active") _application_instance = Application(configuration) if _application_config_path != config_path: raise ValueError("RADICALE_CONFIG must not change: %r != %r" % diff --git a/radicale/__main__.py b/radicale/__main__.py index f2f157b9..25d2b853 100644 --- a/radicale/__main__.py +++ b/radicale/__main__.py @@ -175,7 +175,7 @@ def run() -> None: default_config_active = False if default_config_active: - logger.warn("%s", "No config file found/readable - only default config is active") + logger.warning("%s", "No config file found/readable - only default config is active") if args_ns.verify_storage: logger.info("Verifying storage") @@ -183,7 +183,7 @@ def run() -> None: storage_ = storage.load(configuration) with storage_.acquire_lock("r"): if not storage_.verify(): - logger.critical("Storage verifcation failed") + logger.critical("Storage verification failed") sys.exit(1) except Exception as e: logger.critical("An exception occurred during storage " diff --git a/radicale/app/__init__.py b/radicale/app/__init__.py index 3b8d1800..5fe71d30 100644 --- a/radicale/app/__init__.py +++ b/radicale/app/__init__.py @@ -232,7 +232,7 @@ class Application(ApplicationPartDelete, ApplicationPartHead, path.rstrip("/").endswith("/.well-known/carddav")): return response(*httputils.redirect( base_prefix + "/", client.MOVED_PERMANENTLY)) - # Return NOT FOUND for all other paths containing ".well-knwon" + # Return NOT FOUND for all other paths containing ".well-known" if path.endswith("/.well-known") or "/.well-known/" in path: return response(*httputils.NOT_FOUND) diff --git a/radicale/app/propfind.py b/radicale/app/propfind.py index b1cfc197..009c61dc 100644 --- a/radicale/app/propfind.py +++ b/radicale/app/propfind.py @@ -322,13 +322,13 @@ def xml_propfind_response( responses[404 if is404 else 200].append(element) - for status_code, childs in responses.items(): - if not childs: + for status_code, children in responses.items(): + if not children: continue propstat = ET.Element(xmlutils.make_clark("D:propstat")) response.append(propstat) prop = ET.Element(xmlutils.make_clark("D:prop")) - prop.extend(childs) + prop.extend(children) propstat.append(prop) status = ET.Element(xmlutils.make_clark("D:status")) status.text = xmlutils.make_response(status_code) diff --git a/radicale/app/report.py b/radicale/app/report.py index 9bca7f45..7ea34d21 100644 --- a/radicale/app/report.py +++ b/radicale/app/report.py @@ -363,7 +363,7 @@ def _make_vobject_expanded_item( if hasattr(item.vobject_item.vevent, 'rrule'): rruleset = vevent.getrruleset() - # There is something strage behavour during serialization native datetime, so converting manualy + # There is something strange behaviour during serialization native datetime, so converting manually vevent.dtstart.value = vevent.dtstart.value.strftime(dt_format) if dt_end is not None: vevent.dtend.value = vevent.dtend.value.strftime(dt_format) diff --git a/radicale/auth/__init__.py b/radicale/auth/__init__.py index 15d91ec5..296dbfa8 100644 --- a/radicale/auth/__init__.py +++ b/radicale/auth/__init__.py @@ -52,6 +52,7 @@ def load(configuration: "config.Configuration") -> "BaseAuth": class BaseAuth: _lc_username: bool + _strip_domain: bool def __init__(self, configuration: "config.Configuration") -> None: """Initialize BaseAuth. @@ -63,6 +64,7 @@ class BaseAuth: """ self.configuration = configuration self._lc_username = configuration.get("auth", "lc_username") + self._strip_domain = configuration.get("auth", "strip_domain") def get_external_login(self, environ: types.WSGIEnviron) -> Union[ Tuple[()], Tuple[str, str]]: @@ -91,4 +93,8 @@ class BaseAuth: raise NotImplementedError def login(self, login: str, password: str) -> str: - return self._login(login, password).lower() if self._lc_username else self._login(login, password) + if self._lc_username: + login = login.lower() + if self._strip_domain: + login = login.split('@')[0] + return self._login(login, password) diff --git a/radicale/auth/htpasswd.py b/radicale/auth/htpasswd.py index 689bb0fb..7422e16d 100644 --- a/radicale/auth/htpasswd.py +++ b/radicale/auth/htpasswd.py @@ -36,7 +36,7 @@ pointed to by the ``htpasswd_filename`` configuration value while assuming the password encryption method specified via the ``htpasswd_encryption`` configuration value. -The following htpasswd password encrpytion methods are supported by Radicale +The following htpasswd password encryption methods are supported by Radicale out-of-the-box: - plain-text (created by htpasswd -p ...) -- INSECURE - MD5-APR1 (htpasswd -m ...) -- htpasswd's default method, INSECURE diff --git a/radicale/config.py b/radicale/config.py index b6ab138d..0515813b 100644 --- a/radicale/config.py +++ b/radicale/config.py @@ -191,6 +191,10 @@ DEFAULT_CONFIG_SCHEMA: types.CONFIG_SCHEMA = OrderedDict([ "value": "1", "help": "incorrect authentication delay", "type": positive_float}), + ("strip_domain", { + "value": "False", + "help": "strip domain from username", + "type": bool}), ("lc_username", { "value": "False", "help": "convert username to lowercase, must be true for case-insensitive auth providers", diff --git a/radicale/hook/__init__.py b/radicale/hook/__init__.py index dc6b74c5..e31befc1 100644 --- a/radicale/hook/__init__.py +++ b/radicale/hook/__init__.py @@ -3,14 +3,23 @@ from enum import Enum from typing import Sequence from radicale import pathutils, utils +from radicale.log import logger INTERNAL_TYPES: Sequence[str] = ("none", "rabbitmq") def load(configuration): """Load the storage module chosen in configuration.""" - return utils.load_plugin( - INTERNAL_TYPES, "hook", "Hook", BaseHook, configuration) + try: + return utils.load_plugin( + INTERNAL_TYPES, "hook", "Hook", BaseHook, configuration) + except Exception as e: + logger.warn(e) + logger.warn("Hook \"%s\" failed to load, falling back to \"none\"." % configuration.get("hook", "type")) + configuration = configuration.copy() + configuration.update({"hook": {"type": "none"}}, "hook", privileged=True) + return utils.load_plugin( + INTERNAL_TYPES, "hook", "Hook", BaseHook, configuration) class BaseHook: diff --git a/radicale/item/__init__.py b/radicale/item/__init__.py index c5a32690..a05304ff 100644 --- a/radicale/item/__init__.py +++ b/radicale/item/__init__.py @@ -49,6 +49,12 @@ def read_components(s: str) -> List[vobject.base.Component]: s = re.sub(r"^(PHOTO(?:;[^:\r\n]*)?;ENCODING=b(?:;[^:\r\n]*)?:)" r"data:[^;,\r\n]*;base64,", r"\1", s, flags=re.MULTILINE | re.IGNORECASE) + # Workaround for bug with malformed ICS files containing control codes + # Filter out all control codes except those we expect to find: + # * 0x09 Horizontal Tab + # * 0x0A Line Feed + # * 0x0D Carriage Return + s = re.sub(r'[\x00-\x08\x0B\x0C\x0E-\x1F]', '', s) return list(vobject.readComponents(s, allowQP=True)) @@ -298,7 +304,7 @@ def find_time_range(vobject_item: vobject.base.Component, tag: str Returns a tuple (``start``, ``end``) where ``start`` and ``end`` are POSIX timestamps. - This is intened to be used for matching against simplified prefilters. + This is intended to be used for matching against simplified prefilters. """ if not tag: diff --git a/radicale/item/filter.py b/radicale/item/filter.py index ef6db2f5..cb3e8cdb 100644 --- a/radicale/item/filter.py +++ b/radicale/item/filter.py @@ -241,7 +241,7 @@ def visit_time_ranges(vobject_item: vobject.base.Component, child_name: str, """ - # HACK: According to rfc5545-3.8.4.4 an recurrance that is resheduled + # HACK: According to rfc5545-3.8.4.4 a recurrence that is rescheduled # with Recurrence ID affects the recurrence itself and all following # recurrences too. This is not respected and client don't seem to bother # either. diff --git a/radicale/rights/from_file.py b/radicale/rights/from_file.py index 3b8ede05..d766d1dd 100644 --- a/radicale/rights/from_file.py +++ b/radicale/rights/from_file.py @@ -22,7 +22,7 @@ config (section "rights", key "file"). The login is matched against the "user" key, and the collection path is matched against the "collection" key. In the "collection" regex you can use `{user}` and get groups from the "user" regex with `{0}`, `{1}`, etc. -In consequence of the parameter subsitution you have to write `{{` and `}}` +In consequence of the parameter substitution you have to write `{{` and `}}` if you want to use regular curly braces in the "user" and "collection" regexes. For example, for the "user" key, ".+" means "authenticated user" and ".*" diff --git a/radicale/server.py b/radicale/server.py index 82e5a0b7..600a31ac 100644 --- a/radicale/server.py +++ b/radicale/server.py @@ -291,7 +291,7 @@ def serve(configuration: config.Configuration, try: getaddrinfo = socket.getaddrinfo(address_port[0], address_port[1], 0, socket.SOCK_STREAM, socket.IPPROTO_TCP) except OSError as e: - logger.warn("cannot retrieve IPv4 or IPv6 address of '%s': %s" % (format_address(address_port), e)) + logger.warning("cannot retrieve IPv4 or IPv6 address of '%s': %s" % (format_address(address_port), e)) continue logger.debug("getaddrinfo of '%s': %s" % (format_address(address_port), getaddrinfo)) for (address_family, socket_kind, socket_proto, socket_flags, socket_address) in getaddrinfo: @@ -299,7 +299,7 @@ def serve(configuration: config.Configuration, try: server = server_class(configuration, address_family, (socket_address[0], socket_address[1]), RequestHandler) except OSError as e: - logger.warn("cannot create server socket on '%s': %s" % (format_address(socket_address), e)) + logger.warning("cannot create server socket on '%s': %s" % (format_address(socket_address), e)) continue servers[server.socket] = server server.set_app(application) diff --git a/radicale/storage/multifilesystem/get.py b/radicale/storage/multifilesystem/get.py index c52c1f7e..f5d25816 100644 --- a/radicale/storage/multifilesystem/get.py +++ b/radicale/storage/multifilesystem/get.py @@ -84,7 +84,7 @@ class CollectionPartGet(CollectionPartCache, CollectionPartLock, cache_content = self._load_item_cache(href, cache_hash) if cache_content is None: with self._acquire_cache_lock("item"): - # Lock the item cache to prevent multpile processes from + # Lock the item cache to prevent multiple processes from # generating the same data in parallel. # This improves the performance for multiple requests. if self._storage._lock.locked == "r": @@ -127,7 +127,7 @@ class CollectionPartGet(CollectionPartCache, CollectionPartLock, def get_multi(self, hrefs: Iterable[str] ) -> Iterator[Tuple[str, Optional[radicale_item.Item]]]: - # It's faster to check for file name collissions here, because + # It's faster to check for file name collisions here, because # we only need to call os.listdir once. files = None for href in hrefs: @@ -146,7 +146,7 @@ class CollectionPartGet(CollectionPartCache, CollectionPartLock, def get_all(self) -> Iterator[radicale_item.Item]: for href in self._list(): - # We don't need to check for collissions, because the file names + # We don't need to check for collisions, because the file names # are from os.listdir. item = self._get(href, verify_href=False) if item is not None: diff --git a/radicale/tests/__init__.py b/radicale/tests/__init__.py index f335fd3b..ceb155b4 100644 --- a/radicale/tests/__init__.py +++ b/radicale/tests/__init__.py @@ -112,7 +112,7 @@ class BaseTest: for response in xml.findall(xmlutils.make_clark("D:response")): href = response.find(xmlutils.make_clark("D:href")) assert href.text not in path_responses - prop_respones: Dict[str, Tuple[int, ET.Element]] = {} + prop_responses: Dict[str, Tuple[int, ET.Element]] = {} for propstat in response.findall( xmlutils.make_clark("D:propstat")): status = propstat.find(xmlutils.make_clark("D:status")) @@ -121,16 +121,16 @@ class BaseTest: for element in propstat.findall( "./%s/*" % xmlutils.make_clark("D:prop")): human_tag = xmlutils.make_human_tag(element.tag) - assert human_tag not in prop_respones - prop_respones[human_tag] = (status_code, element) + assert human_tag not in prop_responses + prop_responses[human_tag] = (status_code, element) status = response.find(xmlutils.make_clark("D:status")) if status is not None: - assert not prop_respones + assert not prop_responses assert status.text.startswith("HTTP/1.1 ") status_code = int(status.text.split(" ")[1]) path_responses[href.text] = status_code else: - path_responses[href.text] = prop_respones + path_responses[href.text] = prop_responses return path_responses @staticmethod diff --git a/radicale/tests/test_auth.py b/radicale/tests/test_auth.py index 858e0827..3604e2f9 100644 --- a/radicale/tests/test_auth.py +++ b/radicale/tests/test_auth.py @@ -115,6 +115,16 @@ class TestBaseAuthRequests(BaseTest): def test_htpasswd_comment(self) -> None: self._test_htpasswd("plain", "#comment\n #comment\n \ntmp:bepo\n\n") + def test_htpasswd_lc_username(self) -> None: + self.configure({"auth": {"lc_username": "True"}}) + self._test_htpasswd("plain", "tmp:bepo", ( + ("tmp", "bepo", True), ("TMP", "bepo", True), ("tmp1", "bepo", False))) + + def test_htpasswd_strip_domain(self) -> None: + self.configure({"auth": {"strip_domain": "True"}}) + self._test_htpasswd("plain", "tmp:bepo", ( + ("tmp", "bepo", True), ("tmp@domain.example", "bepo", True), ("tmp1", "bepo", False))) + def test_remote_user(self) -> None: self.configure({"auth": {"type": "remote_user"}}) _, responses = self.propfind("/", """\ diff --git a/radicale/tests/test_base.py b/radicale/tests/test_base.py index b7edb586..ae3669d0 100644 --- a/radicale/tests/test_base.py +++ b/radicale/tests/test_base.py @@ -360,7 +360,7 @@ permissions: RrWw""") self.get(path1, check=404) self.get(path2) - def test_move_between_colections(self) -> None: + def test_move_between_collections(self) -> None: """Move a item.""" self.mkcalendar("/calendar1.ics/") self.mkcalendar("/calendar2.ics/") @@ -373,7 +373,7 @@ permissions: RrWw""") self.get(path1, check=404) self.get(path2) - def test_move_between_colections_duplicate_uid(self) -> None: + def test_move_between_collections_duplicate_uid(self) -> None: """Move a item to a collection which already contains the UID.""" self.mkcalendar("/calendar1.ics/") self.mkcalendar("/calendar2.ics/") @@ -389,7 +389,7 @@ permissions: RrWw""") assert xml.tag == xmlutils.make_clark("D:error") assert xml.find(xmlutils.make_clark("C:no-uid-conflict")) is not None - def test_move_between_colections_overwrite(self) -> None: + def test_move_between_collections_overwrite(self) -> None: """Move a item to a collection which already contains the item.""" self.mkcalendar("/calendar1.ics/") self.mkcalendar("/calendar2.ics/") @@ -403,8 +403,8 @@ permissions: RrWw""") self.request("MOVE", path1, check=204, HTTP_OVERWRITE="T", HTTP_DESTINATION="http://127.0.0.1/"+path2) - def test_move_between_colections_overwrite_uid_conflict(self) -> None: - """Move a item to a collection which already contains the item with + def test_move_between_collections_overwrite_uid_conflict(self) -> None: + """Move an item to a collection which already contains the item with a different UID.""" self.mkcalendar("/calendar1.ics/") self.mkcalendar("/calendar2.ics/") diff --git a/setup.cfg b/setup.cfg index 4326cd97..94a39915 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,26 +1,31 @@ [tool:pytest] -addopts = --typeguard-packages=radicale [tox:tox] +min_version = 4.0 +envlist = py, flake8, isort, mypy [testenv] -extras = test +extras = + test deps = - flake8 - isort - # mypy installation fails with pypy<3.9 - mypy; implementation_name!='pypy' or python_version>='3.9' - types-setuptools + pytest pytest-cov -commands = - flake8 . - isort --check --diff . - # Run mypy if it's installed - python -c 'import importlib.util, subprocess, sys; \ - importlib.util.find_spec("mypy") \ - and sys.exit(subprocess.run(["mypy", "."]).returncode) \ - or print("Skipped: mypy is not installed")' - pytest -r s --cov --cov-report=term --cov-report=xml . +commands = pytest -r s --cov --cov-report=term --cov-report=xml . + +[testenv:flake8] +deps = flake8==7.1.0 +commands = flake8 . +skip_install = True + +[testenv:isort] +deps = isort==5.13.2 +commands = isort --check --diff . +skip_install = True + +[testenv:mypy] +deps = mypy==1.11.0 +commands = mypy . +skip_install = True [tool:isort] known_standard_library = _dummy_thread,_thread,abc,aifc,argparse,array,ast,asynchat,asyncio,asyncore,atexit,audioop,base64,bdb,binascii,binhex,bisect,builtins,bz2,cProfile,calendar,cgi,cgitb,chunk,cmath,cmd,code,codecs,codeop,collections,colorsys,compileall,concurrent,configparser,contextlib,contextvars,copy,copyreg,crypt,csv,ctypes,curses,dataclasses,datetime,dbm,decimal,difflib,dis,distutils,doctest,dummy_threading,email,encodings,ensurepip,enum,errno,faulthandler,fcntl,filecmp,fileinput,fnmatch,formatter,fpectl,fractions,ftplib,functools,gc,getopt,getpass,gettext,glob,grp,gzip,hashlib,heapq,hmac,html,http,imaplib,imghdr,imp,importlib,inspect,io,ipaddress,itertools,json,keyword,lib2to3,linecache,locale,logging,lzma,macpath,mailbox,mailcap,marshal,math,mimetypes,mmap,modulefinder,msilib,msvcrt,multiprocessing,netrc,nis,nntplib,ntpath,numbers,operator,optparse,os,ossaudiodev,parser,pathlib,pdb,pickle,pickletools,pipes,pkgutil,platform,plistlib,poplib,posix,posixpath,pprint,profile,pstats,pty,pwd,py_compile,pyclbr,pydoc,queue,quopri,random,re,readline,reprlib,resource,rlcompleter,runpy,sched,secrets,select,selectors,shelve,shlex,shutil,signal,site,smtpd,smtplib,sndhdr,socket,socketserver,spwd,sqlite3,sre,sre_compile,sre_constants,sre_parse,ssl,stat,statistics,string,stringprep,struct,subprocess,sunau,symbol,symtable,sys,sysconfig,syslog,tabnanny,tarfile,telnetlib,tempfile,termios,test,textwrap,threading,time,timeit,tkinter,token,tokenize,trace,traceback,tracemalloc,tty,turtle,turtledemo,types,typing,unicodedata,unittest,urllib,uu,uuid,venv,warnings,wave,weakref,webbrowser,winreg,winsound,wsgiref,xdrlib,xml,xmlrpc,zipapp,zipfile,zipimport,zlib diff --git a/setup.py b/setup.py index f5d906a2..68e36398 100644 --- a/setup.py +++ b/setup.py @@ -38,9 +38,9 @@ web_files = ["web/internal_data/css/icon.png", install_requires = ["defusedxml", "passlib", "vobject>=0.9.6", "python-dateutil>=2.7.3", "pika>=1.1.0", - "setuptools; python_version<'3.9'"] + ] bcrypt_requires = ["bcrypt"] -test_requires = ["pytest>=7", "typeguard<4.3", "waitress", *bcrypt_requires] +test_requires = ["pytest>=7", "waitress", *bcrypt_requires] setup( name="Radicale", @@ -75,6 +75,7 @@ setup( "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", "Programming Language :: Python :: Implementation :: CPython", "Programming Language :: Python :: Implementation :: PyPy", "Topic :: Office/Business :: Groupware"])