diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index d3baea4..7d1be48 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -40,10 +40,10 @@ caldav's test helper is told, so that port must be free. The CalDAV conformance suite and the readable stories in `tests/test_scenarios.py` (which also run on the files and memory backends) can also run against the docker and external servers of -caldav's own test framework. That framework is not in caldav's wheel, so -it needs a caldav source checkout: an editable caldav install is found by -itself, otherwise set `CALDAV_SOURCE`. The servers are not started for -you; start them in the checkout first: +caldav's own test framework, `caldav.testing`. That needs a caldav +from after [caldav issue 817](https://github.com/python-caldav/caldav/issues/817) +was fixed. The servers are not started for you; start them in a caldav +checkout first: ``` (cd ~/caldav/tests/docker-test-servers/baikal && ./start.sh) @@ -52,12 +52,15 @@ pytest --caldav-servers=Baikal -k Baikal `--caldav-servers` takes a comma-separated list of names, and a named server that is not running fails its tests. `all` takes every server -caldav knows of: the docker servers, the external ones in the checkout's -`tests/caldav_test_servers.yaml` (or `~/.config/caldav/test_servers.yaml`), -and the sections of your caldav configuration marked `test_allowed`. -caldav itself reads only the first of those sources it finds, so the -conftest adds the `test_allowed` sections -([caldav issue 817](https://github.com/python-caldav/caldav/issues/817)). +caldav knows of: the docker servers, the external ones in +`~/.config/caldav/test_servers.yaml` (and in `tests/caldav_test_servers.yaml` +when caldav is installed from a checkout), and the sections of your +caldav configuration marked `test_allowed`. +`--caldav-servers-config=PATH` adds the servers of another such yaml +file, and its entries win over those caldav found itself. Give both +options with `=`: with a space, pytest takes the value for a test path, +moves its root directory there, never loads `tests/conftest.py` and +stops with "unrecognized arguments". Under `all`, a server that is not running is skipped. Without docker, caldav lists a docker server only while it is running. For these servers, the `PYTHON_CALDAV_TEST_*` switches and caldav's per-server @@ -65,6 +68,17 @@ locks apply, so a caldav test run on the same server is waited for rather than clobbered; the embedded Xandikos and Radicale always run, and take no lock. +A run on every server takes hours, most of them on the slowest +servers. Run the servers in parallel, spread over the workers, one +server per worker at a time: + +``` +pytest --caldav-servers=all -n 12 --dist loadgroup +``` + +`--dist loadgroup` is what keeps a server's tests together; plain `-n` +spreads them, and two workers then fight over one server. + ## Sync and async code The async code in the `_async/` directories is the source; the `_sync/` diff --git a/pyproject.toml b/pyproject.toml index 64579ad..52aa7d8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -42,6 +42,7 @@ dev = [ "calendaring[caldav,test-servers]", "pytest>=8.0", "pytest-asyncio>=1.0", + "pytest-xdist>=3.5", "unasync==0.6.0", "ruff==0.17.0", "mypy==2.4.0", @@ -120,7 +121,11 @@ ignore_missing_imports = true testpaths = ["tests"] python_files = ["test_*.py"] # Treat warnings as errors - tests should run clean (roadmap 1.3 demands it too) -filterwarnings = ["error"] +filterwarnings = [ + "error", + # A test server configured with ssl_verify_cert: false (caldav's Zimbra) + "ignore:Unverified HTTPS request:Warning", +] norecursedirs = ["prototypes", "docs", "build"] # The async tests need no marker, so the generated sync copy carries none. asyncio_mode = "auto" diff --git a/tests/conftest.py b/tests/conftest.py index 1b03f8e..30b56ef 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -4,26 +4,18 @@ when they are not installed (``pip install -e .[caldav,test-servers]``). ``--caldav-servers=Baikal,Nextcloud`` (or ``all``) adds the docker and -external servers of caldav's own test framework. That framework is not -in caldav's wheel, so it is loaded from a caldav source checkout: -``$CALDAV_SOURCE``, or the one an editable caldav install points at. -Servers are never started here: start them in the checkout +external servers of caldav's own test framework, ``caldav.testing``. +Servers are never started here: start them in a caldav checkout (``tests/docker-test-servers//start.sh``) before the run. """ from __future__ import annotations -import contextlib -import importlib.util -import os import socket -import sys from collections.abc import Iterator from dataclasses import dataclass from pathlib import Path -from types import ModuleType from typing import Any -from urllib.parse import urlsplit import pytest @@ -71,42 +63,14 @@ def pytest_addoption(parser: pytest.Parser) -> None: "--caldav-servers", default="", metavar="NAMES", - help="also test against caldav's test servers: comma-separated names (Baikal,Nextcloud,...) " - "or 'all'; needs a caldav source checkout ($CALDAV_SOURCE)", + help="also test against caldav's test servers: comma-separated names (Baikal,Nextcloud,...) or 'all'", ) - - -def caldav_checkout() -> Path: - if "CALDAV_SOURCE" in os.environ: - return Path(os.environ["CALDAV_SOURCE"]).expanduser() - import caldav - - return Path(caldav.__file__).parent.parent - - -def caldav_test_framework() -> ModuleType: - """caldav's ``tests/test_servers`` package, imported under a name that cannot clash with ours. - - A path import, until caldav ships it: https://github.com/python-caldav/caldav/issues/817 - """ - name = "caldav_test_servers" - if name in sys.modules: - return sys.modules[name] - checkout = caldav_checkout() - package = checkout / "tests" / "test_servers" - if not (package / "__init__.py").exists(): - raise pytest.UsageError( - f"--caldav-servers needs a caldav source checkout; none at {checkout} (set CALDAV_SOURCE)" - ) - spec = importlib.util.spec_from_file_location( - name, package / "__init__.py", submodule_search_locations=[str(package)] + parser.addoption( + "--caldav-servers-config", + default=None, + metavar="PATH", + help="a caldav test-server yaml to add to the servers caldav finds itself; it wins on a name clash", ) - assert spec is not None - assert spec.loader is not None - module = importlib.util.module_from_spec(spec) - sys.modules[name] = module - spec.loader.exec_module(module) - return module def caldav_test_servers(config: pytest.Config) -> dict[str, Any]: @@ -114,21 +78,27 @@ def caldav_test_servers(config: pytest.Config) -> dict[str, Any]: wanted = config.getoption("--caldav-servers") if not wanted: return {} - framework = caldav_test_framework() - # caldav reads tests/caldav_test_servers.yaml relative to the working directory. - with contextlib.chdir(caldav_checkout()): - registry = framework.get_registry() - # caldav takes the first config source it finds; add the test_allowed sections of the - # caldav configuration even when a test-server file exists (caldav issue 817). - from caldav.config import get_all_test_servers - - allowed = get_all_test_servers() - known = {s.name.lower() for s in registry.all_servers()} - registry.load_from_config( - {n: {"type": "external", "enabled": True, **c} for n, c in allowed.items() if n.lower() not in known} - ) + try: + from caldav.testing.base import EmbeddedTestServer + from caldav.testing.registry import get_registry + except ImportError as e: + raise pytest.UsageError(f"--caldav-servers needs a caldav with the test-server registry: {e}") from e + registry = get_registry() + if path := config.getoption("--caldav-servers-config"): + from caldav.testing.config_loader import load_test_server_config + + # caldav falls back to its default sources when the file is missing + if not Path(path).is_file(): + raise pytest.UsageError(f"--caldav-servers-config: no file {path}") + extra = load_test_server_config(path) + # The registry keeps a server it already has and ignores the file's entry + known = {s.name.lower(): s.name for s in registry.all_servers()} + for name in extra: + if name.lower() in known: + registry.unregister(known[name.lower()]) + registry.load_from_config(extra) registered = registry.enabled_servers() - available = {s.name: s for s in registered if not isinstance(s, framework.EmbeddedTestServer)} + available = {s.name: s for s in registered if not isinstance(s, EmbeddedTestServer)} if wanted == "all": return available names = [n.strip() for n in wanted.split(",") if n.strip()] @@ -158,6 +128,26 @@ def pytest_generate_tests(metafunc: pytest.Metafunc) -> None: metafunc.parametrize("caldav_server", caldav_server_names(metafunc.config), indirect=True, scope="session") +@pytest.hookimpl(tryfirst=True) # before xdist reads the groups +def pytest_collection_modifyitems(config: pytest.Config, items: list[pytest.Item]) -> None: + """Under ``pytest -n N --dist loadgroup``, keep each server's tests on one worker. + + A server then has one connection and one session lock, and the servers run in parallel; + the slowest server sets the pace instead of the sum of them all. + """ + if not config.pluginmanager.hasplugin("xdist"): + return + servers = set(caldav_server_names(config)) + for item in items: + params = getattr(item, "callspec", None) + for name in (params.params.get(key) for key in ("caldav_server", "where")) if params else (): + if name in servers: + item.add_marker(pytest.mark.xdist_group(name)) + # Radicale always listens on 5232, and stop() shuts down whichever one answers there + if "radicale_server" in getattr(item, "fixturenames", ()): + item.add_marker(pytest.mark.xdist_group("Radicale")) + + @dataclass class CalDAVServer: """A running server, and what ``Backend.open(url, kind="caldav", **options)`` needs to reach it.""" @@ -167,17 +157,6 @@ class CalDAVServer: options: dict[str, Any] -def reachable(url: str) -> bool: - """Whether anything answers at the URL's host and port; caldav takes an external server's word for it.""" - parts = urlsplit(url) - port = parts.port or (443 if parts.scheme == "https" else 80) - try: - socket.create_connection((parts.hostname, port), timeout=5).close() - except OSError: - return False - return True - - def running_caldav_server(request: pytest.FixtureRequest, name: str) -> CalDAVServer: """The server by name; a docker or external one is never started, only checked to be up.""" if name in EMBEDDED: @@ -185,8 +164,7 @@ def running_caldav_server(request: pytest.FixtureRequest, name: str) -> CalDAVSe else: server = request.config.stash[SERVERS][name] server.claim() # waits while a caldav test run is using the same server - # An external server without a URL gets one from its auto-connect hints, in caldav. - if not (server.is_accessible() and (not server.url or reachable(server.url))): + if not server.is_accessible(): message = f"{name} is not running at {server.url}; start it from the caldav checkout first" if request.config.getoption("--caldav-servers") == "all": pytest.skip(message)