test: caldav's test servers from caldav.testing, run in parallel

CONTRIBUTING.md

@@ -40,10 +40,10 @@ caldav's test helper is told, so that port must be free.
 The CalDAV conformance suite and the readable stories in
 `tests/test_scenarios.py` (which also run on the files and memory
 backends) can also run against the docker and external servers of
-caldav's own test framework.  That framework is not in caldav's wheel, so
-it needs a caldav source checkout: an editable caldav install is found by
-itself, otherwise set `CALDAV_SOURCE`.  The servers are not started for
-you; start them in the checkout first:
+caldav's own test framework, `caldav.testing`.  That needs a caldav
+from after [caldav issue 817](https://github.com/python-caldav/caldav/issues/817)
+was fixed.  The servers are not started for you; start them in a caldav
+checkout first:
 
 ```
 (cd ~/caldav/tests/docker-test-servers/baikal && ./start.sh)
@@ -52,19 +52,33 @@ pytest --caldav-servers=Baikal -k Baikal
 
 `--caldav-servers` takes a comma-separated list of names, and a named
 server that is not running fails its tests.  `all` takes every server
-caldav knows of: the docker servers, the external ones in the checkout's
-`tests/caldav_test_servers.yaml` (or `~/.config/caldav/test_servers.yaml`),
-and the sections of your caldav configuration marked `test_allowed`.
-caldav itself reads only the first of those sources it finds, so the
-conftest adds the `test_allowed` sections
-([caldav issue 817](https://github.com/python-caldav/caldav/issues/817)).
+caldav knows of: the docker servers, the external ones in
+`~/.config/caldav/test_servers.yaml` (and in `tests/caldav_test_servers.yaml`
+when caldav is installed from a checkout), and the sections of your
+caldav configuration marked `test_allowed`.
+`--caldav-servers-config=PATH` adds the servers of another such yaml
+file, and its entries win over those caldav found itself.  Give both
+options with `=`: with a space, pytest takes the value for a test path,
+moves its root directory there, never loads `tests/conftest.py` and
+stops with "unrecognized arguments".
 Under `all`, a server that is not running is skipped.  Without docker,
 caldav lists a docker server only while it is running.  For these
 servers, the `PYTHON_CALDAV_TEST_*` switches and caldav's per-server
 locks apply, so a caldav test run on the same server is waited for rather
 than clobbered; the embedded Xandikos and Radicale always run, and take
 no lock.
 
+A run on every server takes hours, most of them on the slowest
+servers.  Run the servers in parallel, spread over the workers, one
+server per worker at a time:
+
+```
+pytest --caldav-servers=all -n 12 --dist loadgroup
+```
+
+`--dist loadgroup` is what keeps a server's tests together; plain `-n`
+spreads them, and two workers then fight over one server.
+
 ## Sync and async code
 
 The async code in the `_async/` directories is the source; the `_sync/`

pyproject.toml

@@ -42,6 +42,7 @@ dev = [
   "calendaring[caldav,test-servers]",
   "pytest>=8.0",
   "pytest-asyncio>=1.0",
+  "pytest-xdist>=3.5",
   "unasync==0.6.0",
   "ruff==0.17.0",
   "mypy==2.4.0",
@@ -120,7 +121,11 @@ ignore_missing_imports = true
 testpaths = ["tests"]
 python_files = ["test_*.py"]
 # Treat warnings as errors - tests should run clean (roadmap 1.3 demands it too)
-filterwarnings = ["error"]
+filterwarnings = [
+    "error",
+    # A test server configured with ssl_verify_cert: false (caldav's Zimbra)
+    "ignore:Unverified HTTPS request:Warning",
+]
 norecursedirs = ["prototypes", "docs", "build"]
 # The async tests need no marker, so the generated sync copy carries none.
 asyncio_mode = "auto"

tests/conftest.py

@@ -4,26 +4,18 @@
 when they are not installed (``pip install -e .[caldav,test-servers]``).
 
 ``--caldav-servers=Baikal,Nextcloud`` (or ``all``) adds the docker and
-external servers of caldav's own test framework.  That framework is not
-in caldav's wheel, so it is loaded from a caldav source checkout:
-``$CALDAV_SOURCE``, or the one an editable caldav install points at.
-Servers are never started here: start them in the checkout
+external servers of caldav's own test framework, ``caldav.testing``.
+Servers are never started here: start them in a caldav checkout
 (``tests/docker-test-servers/<name>/start.sh``) before the run.
 """
 
 from __future__ import annotations
 
-import contextlib
-import importlib.util
-import os
 import socket
-import sys
 from collections.abc import Iterator
 from dataclasses import dataclass
 from pathlib import Path
-from types import ModuleType
 from typing import Any
-from urllib.parse import urlsplit
 
 import pytest
 
@@ -71,64 +63,42 @@ def pytest_addoption(parser: pytest.Parser) -> None:
         "--caldav-servers",
         default="",
         metavar="NAMES",
-        help="also test against caldav's test servers: comma-separated names (Baikal,Nextcloud,...) "
-        "or 'all'; needs a caldav source checkout ($CALDAV_SOURCE)",
+        help="also test against caldav's test servers: comma-separated names (Baikal,Nextcloud,...) or 'all'",
     )
-
-
-def caldav_checkout() -> Path:
-    if "CALDAV_SOURCE" in os.environ:
-        return Path(os.environ["CALDAV_SOURCE"]).expanduser()
-    import caldav
-
-    return Path(caldav.__file__).parent.parent
-
-
-def caldav_test_framework() -> ModuleType:
-    """caldav's ``tests/test_servers`` package, imported under a name that cannot clash with ours.
-
-    A path import, until caldav ships it: https://github.com/python-caldav/caldav/issues/817
-    """
-    name = "caldav_test_servers"
-    if name in sys.modules:
-        return sys.modules[name]
-    checkout = caldav_checkout()
-    package = checkout / "tests" / "test_servers"
-    if not (package / "__init__.py").exists():
-        raise pytest.UsageError(
-            f"--caldav-servers needs a caldav source checkout; none at {checkout} (set CALDAV_SOURCE)"
-        )
-    spec = importlib.util.spec_from_file_location(
-        name, package / "__init__.py", submodule_search_locations=[str(package)]
+    parser.addoption(
+        "--caldav-servers-config",
+        default=None,
+        metavar="PATH",
+        help="a caldav test-server yaml to add to the servers caldav finds itself; it wins on a name clash",
     )
-    assert spec is not None
-    assert spec.loader is not None
-    module = importlib.util.module_from_spec(spec)
-    sys.modules[name] = module
-    spec.loader.exec_module(module)
-    return module
 
 
 def caldav_test_servers(config: pytest.Config) -> dict[str, Any]:
     """The servers ``--caldav-servers`` asked for, by name; the embedded ones are left to the fixtures above."""
     wanted = config.getoption("--caldav-servers")
     if not wanted:
         return {}
-    framework = caldav_test_framework()
-    # caldav reads tests/caldav_test_servers.yaml relative to the working directory.
-    with contextlib.chdir(caldav_checkout()):
-        registry = framework.get_registry()
-    # caldav takes the first config source it finds; add the test_allowed sections of the
-    # caldav configuration even when a test-server file exists (caldav issue 817).
-    from caldav.config import get_all_test_servers
-
-    allowed = get_all_test_servers()
-    known = {s.name.lower() for s in registry.all_servers()}
-    registry.load_from_config(
-        {n: {"type": "external", "enabled": True, **c} for n, c in allowed.items() if n.lower() not in known}
-    )
+    try:
+        from caldav.testing.base import EmbeddedTestServer
+        from caldav.testing.registry import get_registry
+    except ImportError as e:
+        raise pytest.UsageError(f"--caldav-servers needs a caldav with the test-server registry: {e}") from e
+    registry = get_registry()
+    if path := config.getoption("--caldav-servers-config"):
+        from caldav.testing.config_loader import load_test_server_config
+
+        # caldav falls back to its default sources when the file is missing
+        if not Path(path).is_file():
+            raise pytest.UsageError(f"--caldav-servers-config: no file {path}")
+        extra = load_test_server_config(path)
+        # The registry keeps a server it already has and ignores the file's entry
+        known = {s.name.lower(): s.name for s in registry.all_servers()}
+        for name in extra:
+            if name.lower() in known:
+                registry.unregister(known[name.lower()])
+        registry.load_from_config(extra)
     registered = registry.enabled_servers()
-    available = {s.name: s for s in registered if not isinstance(s, framework.EmbeddedTestServer)}
+    available = {s.name: s for s in registered if not isinstance(s, EmbeddedTestServer)}
     if wanted == "all":
         return available
     names = [n.strip() for n in wanted.split(",") if n.strip()]
@@ -158,6 +128,26 @@ def pytest_generate_tests(metafunc: pytest.Metafunc) -> None:
         metafunc.parametrize("caldav_server", caldav_server_names(metafunc.config), indirect=True, scope="session")
 
 
+@pytest.hookimpl(tryfirst=True)  # before xdist reads the groups
+def pytest_collection_modifyitems(config: pytest.Config, items: list[pytest.Item]) -> None:
+    """Under ``pytest -n N --dist loadgroup``, keep each server's tests on one worker.
+
+    A server then has one connection and one session lock, and the servers run in parallel;
+    the slowest server sets the pace instead of the sum of them all.
+    """
+    if not config.pluginmanager.hasplugin("xdist"):
+        return
+    servers = set(caldav_server_names(config))
+    for item in items:
+        params = getattr(item, "callspec", None)
+        for name in (params.params.get(key) for key in ("caldav_server", "where")) if params else ():
+            if name in servers:
+                item.add_marker(pytest.mark.xdist_group(name))
+        # Radicale always listens on 5232, and stop() shuts down whichever one answers there
+        if "radicale_server" in getattr(item, "fixturenames", ()):
+            item.add_marker(pytest.mark.xdist_group("Radicale"))
+
+
 @dataclass
 class CalDAVServer:
     """A running server, and what ``Backend.open(url, kind="caldav", **options)`` needs to reach it."""
@@ -167,26 +157,14 @@ class CalDAVServer:
     options: dict[str, Any]
 
 
-def reachable(url: str) -> bool:
-    """Whether anything answers at the URL's host and port; caldav takes an external server's word for it."""
-    parts = urlsplit(url)
-    port = parts.port or (443 if parts.scheme == "https" else 80)
-    try:
-        socket.create_connection((parts.hostname, port), timeout=5).close()
-    except OSError:
-        return False
-    return True
-
-
 def running_caldav_server(request: pytest.FixtureRequest, name: str) -> CalDAVServer:
     """The server by name; a docker or external one is never started, only checked to be up."""
     if name in EMBEDDED:
         server = request.getfixturevalue(f"{name.lower()}_server")
     else:
         server = request.config.stash[SERVERS][name]
         server.claim()  # waits while a caldav test run is using the same server
-        # An external server without a URL gets one from its auto-connect hints, in caldav.
-        if not (server.is_accessible() and (not server.url or reachable(server.url))):
+        if not server.is_accessible():
             message = f"{name} is not running at {server.url}; start it from the caldav checkout first"
             if request.config.getoption("--caldav-servers") == "all":
                 pytest.skip(message)