From 84b899fab7b0f30ea008338d9c5ecbd59eb1309f Mon Sep 17 00:00:00 2001 From: Tobias Brox Date: Sun, 25 Oct 2026 00:25:04 +0200 Subject: [PATCH 1/3] test: use the test servers from caldav.testing caldav now ships its test-server registry in caldav.testing, so the conftest imports it instead of loading tests/test_servers by path from a caldav checkout, and CALDAV_SOURCE is gone. The registry also merges all its config sources, finds the checkout's yaml whatever the working directory, and probes external servers itself, so the conftest's workarounds for those go too. https://github.com/python-caldav/caldav/issues/817 https://github.com/python-caldav/caldav/pull/824 Prompt: Now that we have #817 done on the local caldav master (please confirm), please fix the calendaring library to use this, and run the integration tests towards all the docker servers plus all the external servers defined under [a caldav checkout holding the test servers' passwords] [#817: https://github.com/python-caldav/caldav/issues/817] Assisted-By: Claude Opus 5.5 (1M context) Reviewed-by: Tobias Brox --- CONTRIBUTING.md | 18 +++++----- tests/conftest.py | 85 ++++++----------------------------------------- 2 files changed, 19 insertions(+), 84 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index d3baea4..cd6fe49 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -40,10 +40,10 @@ caldav's test helper is told, so that port must be free. The CalDAV conformance suite and the readable stories in `tests/test_scenarios.py` (which also run on the files and memory backends) can also run against the docker and external servers of -caldav's own test framework. That framework is not in caldav's wheel, so -it needs a caldav source checkout: an editable caldav install is found by -itself, otherwise set `CALDAV_SOURCE`. The servers are not started for -you; start them in the checkout first: +caldav's own test framework, `caldav.testing`. That needs a caldav +from after [caldav issue 817](https://github.com/python-caldav/caldav/issues/817) +was fixed. The servers are not started for you; start them in a caldav +checkout first: ``` (cd ~/caldav/tests/docker-test-servers/baikal && ./start.sh) @@ -52,12 +52,10 @@ pytest --caldav-servers=Baikal -k Baikal `--caldav-servers` takes a comma-separated list of names, and a named server that is not running fails its tests. `all` takes every server -caldav knows of: the docker servers, the external ones in the checkout's -`tests/caldav_test_servers.yaml` (or `~/.config/caldav/test_servers.yaml`), -and the sections of your caldav configuration marked `test_allowed`. -caldav itself reads only the first of those sources it finds, so the -conftest adds the `test_allowed` sections -([caldav issue 817](https://github.com/python-caldav/caldav/issues/817)). +caldav knows of: the docker servers, the external ones in +`~/.config/caldav/test_servers.yaml` (and in `tests/caldav_test_servers.yaml` +when caldav is installed from a checkout), and the sections of your +caldav configuration marked `test_allowed`. Under `all`, a server that is not running is skipped. Without docker, caldav lists a docker server only while it is running. For these servers, the `PYTHON_CALDAV_TEST_*` switches and caldav's per-server diff --git a/tests/conftest.py b/tests/conftest.py index 1b03f8e..985aecb 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -4,26 +4,17 @@ when they are not installed (``pip install -e .[caldav,test-servers]``). ``--caldav-servers=Baikal,Nextcloud`` (or ``all``) adds the docker and -external servers of caldav's own test framework. That framework is not -in caldav's wheel, so it is loaded from a caldav source checkout: -``$CALDAV_SOURCE``, or the one an editable caldav install points at. -Servers are never started here: start them in the checkout +external servers of caldav's own test framework, ``caldav.testing``. +Servers are never started here: start them in a caldav checkout (``tests/docker-test-servers//start.sh``) before the run. """ from __future__ import annotations -import contextlib -import importlib.util -import os import socket -import sys from collections.abc import Iterator from dataclasses import dataclass -from pathlib import Path -from types import ModuleType from typing import Any -from urllib.parse import urlsplit import pytest @@ -71,64 +62,22 @@ def pytest_addoption(parser: pytest.Parser) -> None: "--caldav-servers", default="", metavar="NAMES", - help="also test against caldav's test servers: comma-separated names (Baikal,Nextcloud,...) " - "or 'all'; needs a caldav source checkout ($CALDAV_SOURCE)", + help="also test against caldav's test servers: comma-separated names (Baikal,Nextcloud,...) or 'all'", ) -def caldav_checkout() -> Path: - if "CALDAV_SOURCE" in os.environ: - return Path(os.environ["CALDAV_SOURCE"]).expanduser() - import caldav - - return Path(caldav.__file__).parent.parent - - -def caldav_test_framework() -> ModuleType: - """caldav's ``tests/test_servers`` package, imported under a name that cannot clash with ours. - - A path import, until caldav ships it: https://github.com/python-caldav/caldav/issues/817 - """ - name = "caldav_test_servers" - if name in sys.modules: - return sys.modules[name] - checkout = caldav_checkout() - package = checkout / "tests" / "test_servers" - if not (package / "__init__.py").exists(): - raise pytest.UsageError( - f"--caldav-servers needs a caldav source checkout; none at {checkout} (set CALDAV_SOURCE)" - ) - spec = importlib.util.spec_from_file_location( - name, package / "__init__.py", submodule_search_locations=[str(package)] - ) - assert spec is not None - assert spec.loader is not None - module = importlib.util.module_from_spec(spec) - sys.modules[name] = module - spec.loader.exec_module(module) - return module - - def caldav_test_servers(config: pytest.Config) -> dict[str, Any]: """The servers ``--caldav-servers`` asked for, by name; the embedded ones are left to the fixtures above.""" wanted = config.getoption("--caldav-servers") if not wanted: return {} - framework = caldav_test_framework() - # caldav reads tests/caldav_test_servers.yaml relative to the working directory. - with contextlib.chdir(caldav_checkout()): - registry = framework.get_registry() - # caldav takes the first config source it finds; add the test_allowed sections of the - # caldav configuration even when a test-server file exists (caldav issue 817). - from caldav.config import get_all_test_servers - - allowed = get_all_test_servers() - known = {s.name.lower() for s in registry.all_servers()} - registry.load_from_config( - {n: {"type": "external", "enabled": True, **c} for n, c in allowed.items() if n.lower() not in known} - ) - registered = registry.enabled_servers() - available = {s.name: s for s in registered if not isinstance(s, framework.EmbeddedTestServer)} + try: + from caldav.testing.base import EmbeddedTestServer + from caldav.testing.registry import get_registry + except ImportError as e: + raise pytest.UsageError(f"--caldav-servers needs a caldav with the test-server registry: {e}") from e + registered = get_registry().enabled_servers() + available = {s.name: s for s in registered if not isinstance(s, EmbeddedTestServer)} if wanted == "all": return available names = [n.strip() for n in wanted.split(",") if n.strip()] @@ -167,17 +116,6 @@ class CalDAVServer: options: dict[str, Any] -def reachable(url: str) -> bool: - """Whether anything answers at the URL's host and port; caldav takes an external server's word for it.""" - parts = urlsplit(url) - port = parts.port or (443 if parts.scheme == "https" else 80) - try: - socket.create_connection((parts.hostname, port), timeout=5).close() - except OSError: - return False - return True - - def running_caldav_server(request: pytest.FixtureRequest, name: str) -> CalDAVServer: """The server by name; a docker or external one is never started, only checked to be up.""" if name in EMBEDDED: @@ -185,8 +123,7 @@ def running_caldav_server(request: pytest.FixtureRequest, name: str) -> CalDAVSe else: server = request.config.stash[SERVERS][name] server.claim() # waits while a caldav test run is using the same server - # An external server without a URL gets one from its auto-connect hints, in caldav. - if not (server.is_accessible() and (not server.url or reachable(server.url))): + if not server.is_accessible(): message = f"{name} is not running at {server.url}; start it from the caldav checkout first" if request.config.getoption("--caldav-servers") == "all": pytest.skip(message) From ed80c6024ce7898bdc297a2e779149309ffafb20 Mon Sep 17 00:00:00 2001 From: Tobias Brox Date: Sun, 25 Oct 2026 00:25:23 +0200 Subject: [PATCH 2/3] test: allow test servers without TLS verification caldav's Zimbra test server is configured with ssl_verify_cert: false, and urllib3's "Unverified HTTPS request" warning then failed every Zimbra test under filterwarnings = error. The filter matches on the message, so it needs no urllib3 import. Prompt: Now that we have #817 done on the local caldav master (please confirm), please fix the calendaring library to use this, and run the integration tests towards all the docker servers plus all the external servers defined under [a caldav checkout holding the test servers' passwords] [#817: https://github.com/python-caldav/caldav/issues/817; this fixes what the Zimbra run turned up] Assisted-By: Claude Opus 5.5 (1M context) Reviewed-by: Tobias Brox --- pyproject.toml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 64579ad..fad5f5b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -120,7 +120,11 @@ ignore_missing_imports = true testpaths = ["tests"] python_files = ["test_*.py"] # Treat warnings as errors - tests should run clean (roadmap 1.3 demands it too) -filterwarnings = ["error"] +filterwarnings = [ + "error", + # A test server configured with ssl_verify_cert: false (caldav's Zimbra) + "ignore:Unverified HTTPS request:Warning", +] norecursedirs = ["prototypes", "docs", "build"] # The async tests need no marker, so the generated sync copy carries none. asyncio_mode = "auto" From 1b06758e7a6d7e314c4676789ae88fccf409cc94 Mon Sep 17 00:00:00 2001 From: Tobias Brox Date: Sun, 25 Oct 2026 08:41:21 +0100 Subject: [PATCH 3/3] test: run caldav's test servers in parallel A run on every caldav test server took hours, most of them on the slowest server. Under pytest -n N --dist loadgroup each server's tests now stay on one worker, so the servers run in parallel and keep one connection and one session lock each; pytest-xdist joins the dev extra. --caldav-servers-config adds the servers of a test-server yaml outside the caldav checkout, its entries winning on a name clash. Prompt: The biggest problem is the purelymail runs [due to test sleeps due to purelymail having an async search cache], a single run can easily take several hoursl If we cannot do that with pytest-xdist, then it's needed to do aome similar lgoic as in [a personal script that runs caldav's tests in parallel, one tmux window per slow server] Followup-Prompt: Do the needful ensuring all tests are run towardes all servers - if ecloud failed on the last run, then it's a good reason to retry, [(sic)] Assisted-By: Claude Opus 5.5 (1M context) Reviewed-by: Tobias Brox --- CONTRIBUTING.md | 16 ++++++++++++++++ pyproject.toml | 1 + tests/conftest.py | 43 ++++++++++++++++++++++++++++++++++++++++++- 3 files changed, 59 insertions(+), 1 deletion(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index cd6fe49..7d1be48 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -56,6 +56,11 @@ caldav knows of: the docker servers, the external ones in `~/.config/caldav/test_servers.yaml` (and in `tests/caldav_test_servers.yaml` when caldav is installed from a checkout), and the sections of your caldav configuration marked `test_allowed`. +`--caldav-servers-config=PATH` adds the servers of another such yaml +file, and its entries win over those caldav found itself. Give both +options with `=`: with a space, pytest takes the value for a test path, +moves its root directory there, never loads `tests/conftest.py` and +stops with "unrecognized arguments". Under `all`, a server that is not running is skipped. Without docker, caldav lists a docker server only while it is running. For these servers, the `PYTHON_CALDAV_TEST_*` switches and caldav's per-server @@ -63,6 +68,17 @@ locks apply, so a caldav test run on the same server is waited for rather than clobbered; the embedded Xandikos and Radicale always run, and take no lock. +A run on every server takes hours, most of them on the slowest +servers. Run the servers in parallel, spread over the workers, one +server per worker at a time: + +``` +pytest --caldav-servers=all -n 12 --dist loadgroup +``` + +`--dist loadgroup` is what keeps a server's tests together; plain `-n` +spreads them, and two workers then fight over one server. + ## Sync and async code The async code in the `_async/` directories is the source; the `_sync/` diff --git a/pyproject.toml b/pyproject.toml index fad5f5b..52aa7d8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -42,6 +42,7 @@ dev = [ "calendaring[caldav,test-servers]", "pytest>=8.0", "pytest-asyncio>=1.0", + "pytest-xdist>=3.5", "unasync==0.6.0", "ruff==0.17.0", "mypy==2.4.0", diff --git a/tests/conftest.py b/tests/conftest.py index 985aecb..30b56ef 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -14,6 +14,7 @@ import socket from collections.abc import Iterator from dataclasses import dataclass +from pathlib import Path from typing import Any import pytest @@ -64,6 +65,12 @@ def pytest_addoption(parser: pytest.Parser) -> None: metavar="NAMES", help="also test against caldav's test servers: comma-separated names (Baikal,Nextcloud,...) or 'all'", ) + parser.addoption( + "--caldav-servers-config", + default=None, + metavar="PATH", + help="a caldav test-server yaml to add to the servers caldav finds itself; it wins on a name clash", + ) def caldav_test_servers(config: pytest.Config) -> dict[str, Any]: @@ -76,7 +83,21 @@ def caldav_test_servers(config: pytest.Config) -> dict[str, Any]: from caldav.testing.registry import get_registry except ImportError as e: raise pytest.UsageError(f"--caldav-servers needs a caldav with the test-server registry: {e}") from e - registered = get_registry().enabled_servers() + registry = get_registry() + if path := config.getoption("--caldav-servers-config"): + from caldav.testing.config_loader import load_test_server_config + + # caldav falls back to its default sources when the file is missing + if not Path(path).is_file(): + raise pytest.UsageError(f"--caldav-servers-config: no file {path}") + extra = load_test_server_config(path) + # The registry keeps a server it already has and ignores the file's entry + known = {s.name.lower(): s.name for s in registry.all_servers()} + for name in extra: + if name.lower() in known: + registry.unregister(known[name.lower()]) + registry.load_from_config(extra) + registered = registry.enabled_servers() available = {s.name: s for s in registered if not isinstance(s, EmbeddedTestServer)} if wanted == "all": return available @@ -107,6 +128,26 @@ def pytest_generate_tests(metafunc: pytest.Metafunc) -> None: metafunc.parametrize("caldav_server", caldav_server_names(metafunc.config), indirect=True, scope="session") +@pytest.hookimpl(tryfirst=True) # before xdist reads the groups +def pytest_collection_modifyitems(config: pytest.Config, items: list[pytest.Item]) -> None: + """Under ``pytest -n N --dist loadgroup``, keep each server's tests on one worker. + + A server then has one connection and one session lock, and the servers run in parallel; + the slowest server sets the pace instead of the sum of them all. + """ + if not config.pluginmanager.hasplugin("xdist"): + return + servers = set(caldav_server_names(config)) + for item in items: + params = getattr(item, "callspec", None) + for name in (params.params.get(key) for key in ("caldav_server", "where")) if params else (): + if name in servers: + item.add_marker(pytest.mark.xdist_group(name)) + # Radicale always listens on 5232, and stop() shuts down whichever one answers there + if "radicale_server" in getattr(item, "fixturenames", ()): + item.add_marker(pytest.mark.xdist_group("Radicale")) + + @dataclass class CalDAVServer: """A running server, and what ``Backend.open(url, kind="caldav", **options)`` needs to reach it."""