From 0e702da0ce293bd045499fcb908998d78a595a39 Mon Sep 17 00:00:00 2001 From: whisper67265 Date: Thu, 4 Jun 2026 09:27:40 -0600 Subject: [PATCH] plugin test reliability and stabilization --- .github/README.md | 29 +++++++++++++ .github/workflows/ci-plugin-auth.yml | 2 + .github/workflows/ci-plugin-functional.yml | 1 + .github/workflows/ci-plugin-smoke.yml | 2 + pyproject.toml | 4 +- scripts/README.md | 5 ++- scripts/lib/weblate-stack.sh | 49 +++++++++++++++++++++- scripts/plugin-auth.sh | 8 +++- scripts/plugin-functional.sh | 9 ++-- scripts/plugin-smoke.sh | 10 +++-- tests/plugin/conftest.py | 43 ++++++++++++++++++- tests/plugin/test_functional.py | 10 +++++ tests/plugin/test_rate_limit.py | 3 ++ uv.lock | 15 +++++++ 14 files changed, 177 insertions(+), 13 deletions(-) diff --git a/.github/README.md b/.github/README.md index 5456bd0..382aa1e 100644 --- a/.github/README.md +++ b/.github/README.md @@ -27,6 +27,35 @@ GitHub Actions and CI/CD helpers for this repository. Callable workflows (`ci-*`, `ci-plugin-*`) are triggered only via `workflow_call` from `ci.yml`, not directly on push. +## Plugin integration jobs + +Three callable workflows exercise the live Weblate Docker stack ([`docker/docker-compose.ci.yml`](../docker/docker-compose.ci.yml)). Each job builds the image, runs `compose up -d --wait`, probes `/healthz/` and the Boost ping endpoint, creates an API token (with retry), then runs pytest with `pytest-timeout` and one rerun on failure. + +| Job | Workflow | Typical duration | Hard limit (`timeout-minutes`) | Notes | +|-----|----------|------------------|--------------------------------|-------| +| Plugin smoke | [`ci-plugin-smoke.yml`](workflows/ci-plugin-smoke.yml) | ~8–12 min | 15 | Stack image build dominates | +| Plugin functional | [`ci-plugin-functional.yml`](workflows/ci-plugin-functional.yml) | ~15–22 min | 25 | GitHub E2E needs repository secret | +| Plugin auth | [`ci-plugin-auth.yml`](workflows/ci-plugin-auth.yml) | ~8–12 min | 10 | Auth + rate-limit tests | + +### Secrets and environment + +| Variable | Where | Purpose | +|----------|-------|---------| +| `GH_TEST_REPO_TOKEN` | Repository secret (functional job only) | Classic PAT with `repo` scope for ephemeral GitHub repos in [`tests/plugin/test_functional.py`](../tests/plugin/test_functional.py). If unset, GitHub/Celery E2E tests are skipped. | +| `HEALTH_TIMEOUT` | CI workflow env / shell default | Seconds to wait for `/healthz/` after compose `--wait`. Defaults: smoke/auth **240**, functional **300**. | +| `PYTEST_PLUGIN_OPTS` | Optional override in entrypoint scripts | Default includes `--timeout`, `--timeout-method=thread`, `--reruns 1`, `--reruns-delay 5`. Smoke/auth use `--timeout=120`; functional uses `--timeout=300`. | +| `WEBLATE_PORT` | Optional | Host port for Weblate (default **8080**). | + +### Local reproduction + +```bash +bash scripts/plugin-smoke.sh +bash scripts/plugin-auth.sh +GH_TEST_REPO_TOKEN= bash scripts/plugin-functional.sh +``` + +Skip slow plugin tests during local iteration: add `-m "not slow"` to the pytest invocation in the script, or set `PYTEST_PLUGIN_OPTS` accordingly. + ## Other paths | Path | Role | diff --git a/.github/workflows/ci-plugin-auth.yml b/.github/workflows/ci-plugin-auth.yml index 68db4d3..abf7468 100644 --- a/.github/workflows/ci-plugin-auth.yml +++ b/.github/workflows/ci-plugin-auth.yml @@ -33,6 +33,8 @@ jobs: version: 0.11.12 - name: Run plugin auth tests + env: + HEALTH_TIMEOUT: '240' run: bash scripts/plugin-auth.sh - name: Upload logs on failure diff --git a/.github/workflows/ci-plugin-functional.yml b/.github/workflows/ci-plugin-functional.yml index 29c6857..c638592 100644 --- a/.github/workflows/ci-plugin-functional.yml +++ b/.github/workflows/ci-plugin-functional.yml @@ -45,6 +45,7 @@ jobs: - name: Run plugin functional tests env: GH_TEST_REPO_TOKEN: ${{ secrets.GH_TEST_REPO_TOKEN }} + HEALTH_TIMEOUT: '300' run: bash scripts/plugin-functional.sh - name: Upload logs on failure diff --git a/.github/workflows/ci-plugin-smoke.yml b/.github/workflows/ci-plugin-smoke.yml index 051bccf..ac13f6a 100644 --- a/.github/workflows/ci-plugin-smoke.yml +++ b/.github/workflows/ci-plugin-smoke.yml @@ -33,6 +33,8 @@ jobs: version: 0.11.12 - name: Run plugin smoke tests + env: + HEALTH_TIMEOUT: '240' run: bash scripts/plugin-smoke.sh - name: Upload logs on failure diff --git a/pyproject.toml b/pyproject.toml index 5206a78..1855e71 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -16,6 +16,7 @@ lint = [ ] plugin = [ "pytest==9.0.3", + "pytest-rerunfailures==16.3", "pytest-timeout==2.4.0" ] pre-commit = [ @@ -132,7 +133,8 @@ unauthorized_licenses = [] [tool.pytest.ini_options] addopts = ["-m", "not plugin"] markers = [ - "plugin: requires live Weblate stack (Docker Compose) and optional WEBLATE_API_TOKEN" + "plugin: requires live Weblate stack (Docker Compose) and optional WEBLATE_API_TOKEN", + "slow: long-running plugin integration test" ] python_classes = ["Test*"] python_files = ["test_*.py", "*_test.py"] diff --git a/scripts/README.md b/scripts/README.md index c517d57..f826c6e 100644 --- a/scripts/README.md +++ b/scripts/README.md @@ -24,5 +24,8 @@ bash scripts/plugin-smoke.sh source scripts/lib/weblate-stack.sh stack_build stack_up -stack_wait_healthy 120 +stack_wait_healthy 240 +stack_wait_api_ready ``` + +CI entrypoints use `compose up -d --wait`, `HEALTH_TIMEOUT` (smoke/auth **240**, functional **300**), `stack_create_token_retry`, and `PYTEST_PLUGIN_OPTS` for pytest timeout/reruns. See [`.github/README.md`](../.github/README.md#plugin-integration-jobs). diff --git a/scripts/lib/weblate-stack.sh b/scripts/lib/weblate-stack.sh index d5ec75d..af88bad 100755 --- a/scripts/lib/weblate-stack.sh +++ b/scripts/lib/weblate-stack.sh @@ -16,11 +16,12 @@ stack_build() { } stack_up() { - compose up -d "$@" + # Wait for compose healthchecks (Postgres, Redis, Weblate) before returning. + compose up -d --wait "$@" } stack_wait_healthy() { - local timeout="${1:-120}" + local timeout="${1:-240}" local port="${WEBLATE_PORT:-8080}" local url="http://localhost:${port}/healthz/" local interval=5 @@ -42,6 +43,27 @@ stack_wait_healthy() { return 1 } +stack_wait_api_ready() { + local port="${WEBLATE_PORT:-8080}" + local url="http://localhost:${port}/boost-endpoint/plugin-ping/" + local attempts="${1:-12}" + local interval="${2:-5}" + local i=0 + + echo "Waiting for Boost endpoint at ${url}..." + while [ "$i" -lt "$attempts" ]; do + if curl -sf "$url" > /dev/null 2>&1; then + echo "Boost endpoint is ready (after $((i * interval))s)." + return 0 + fi + sleep "$interval" + i=$((i + 1)) + done + + echo "ERROR: Boost endpoint did not become ready in $((attempts * interval))s." + return 1 +} + stack_ensure_github_known_hosts() { # Weblate git uses GIT_SSH with UserKnownHostsFile=/app/data/ssh/known_hosts # and StrictHostKeyChecking=yes. Celery/component sync over git@github.com fails @@ -79,6 +101,29 @@ t = Token.objects.create(user=u, key=get_token("wlp" if u.is_bot else "wlu")) print(t.key)' } +stack_create_token_retry() { + local user="${1:-admin}" + local attempts="${2:-3}" + local interval="${3:-5}" + local attempt=1 + local token="" + + while [ "$attempt" -le "$attempts" ]; do + if token="$(stack_create_token "$user")" && [ -n "$token" ]; then + echo "$token" + return 0 + fi + if [ "$attempt" -lt "$attempts" ]; then + echo "Token creation attempt ${attempt}/${attempts} failed; retrying in ${interval}s..." >&2 + sleep "$interval" + fi + attempt=$((attempt + 1)) + done + + echo "ERROR: Failed to create API token for user ${user} after ${attempts} attempts." >&2 + return 1 +} + stack_logs() { local file="${1:-}" if [ -n "$file" ]; then diff --git a/scripts/plugin-auth.sh b/scripts/plugin-auth.sh index d129972..8f2b05f 100755 --- a/scripts/plugin-auth.sh +++ b/scripts/plugin-auth.sh @@ -30,10 +30,11 @@ echo "=== Starting stack ===" stack_up echo "=== Waiting for Weblate ===" -stack_wait_healthy "${HEALTH_TIMEOUT:-120}" +stack_wait_healthy "${HEALTH_TIMEOUT:-240}" +stack_wait_api_ready echo "=== Creating API token ===" -WEBLATE_API_TOKEN="$(stack_create_token admin)" +WEBLATE_API_TOKEN="$(stack_create_token_retry admin)" export WEBLATE_API_TOKEN export WEBLATE_LIVE_BASE_URL="${WEBLATE_LIVE_BASE_URL:-http://localhost:${WEBLATE_PORT:-8080}}" export WEBLATE_COMPOSE_FILE="${COMPOSE_FILE}" @@ -43,5 +44,8 @@ export BOOST_ENDPOINT_THROTTLE_ADD_OR_UPDATE="${BOOST_ENDPOINT_THROTTLE_ADD_OR_U echo "=== Running auth tests ===" uv pip install --quiet --system --group plugin +PYTEST_PLUGIN_OPTS="${PYTEST_PLUGIN_OPTS:---timeout=120 --timeout-method=thread --reruns 1 --reruns-delay 5}" +# shellcheck disable=SC2086 python -m pytest --confcutdir=tests/plugin --override-ini addopts= \ + $PYTEST_PLUGIN_OPTS \ tests/plugin/test_auth.py tests/plugin/test_rate_limit.py -v diff --git a/scripts/plugin-functional.sh b/scripts/plugin-functional.sh index dacc6c4..f96c2cc 100755 --- a/scripts/plugin-functional.sh +++ b/scripts/plugin-functional.sh @@ -30,10 +30,11 @@ echo "=== Starting stack ===" stack_up echo "=== Waiting for Weblate ===" -stack_wait_healthy "${HEALTH_TIMEOUT:-180}" +stack_wait_healthy "${HEALTH_TIMEOUT:-300}" +stack_wait_api_ready echo "=== Creating API token ===" -WEBLATE_API_TOKEN="$(stack_create_token admin)" +WEBLATE_API_TOKEN="$(stack_create_token_retry admin)" export WEBLATE_API_TOKEN export WEBLATE_LIVE_BASE_URL="${WEBLATE_LIVE_BASE_URL:-http://localhost:${WEBLATE_PORT:-8080}}" export WEBLATE_COMPOSE_FILE="${COMPOSE_FILE}" @@ -59,5 +60,7 @@ fi echo "=== Running functional tests ===" uv pip install --quiet --system --group plugin +PYTEST_PLUGIN_OPTS="${PYTEST_PLUGIN_OPTS:---timeout=300 --timeout-method=thread --reruns 1 --reruns-delay 5}" +# shellcheck disable=SC2086 python -m pytest --confcutdir=tests/plugin --override-ini addopts= \ - tests/plugin/test_functional.py -v --timeout=300 + $PYTEST_PLUGIN_OPTS tests/plugin/test_functional.py -v diff --git a/scripts/plugin-smoke.sh b/scripts/plugin-smoke.sh index 403b64e..5900fca 100755 --- a/scripts/plugin-smoke.sh +++ b/scripts/plugin-smoke.sh @@ -30,10 +30,11 @@ echo "=== Starting stack ===" stack_up echo "=== Waiting for Weblate ===" -stack_wait_healthy "${HEALTH_TIMEOUT:-120}" +stack_wait_healthy "${HEALTH_TIMEOUT:-240}" +stack_wait_api_ready echo "=== Creating API token ===" -WEBLATE_API_TOKEN="$(stack_create_token admin)" +WEBLATE_API_TOKEN="$(stack_create_token_retry admin)" export WEBLATE_API_TOKEN export WEBLATE_LIVE_BASE_URL="${WEBLATE_LIVE_BASE_URL:-http://localhost:${WEBLATE_PORT:-8080}}" export WEBLATE_COMPOSE_FILE="${COMPOSE_FILE}" @@ -44,4 +45,7 @@ echo "=== Running smoke tests ===" # --system: setup-python on CI has no project venv (matches ci-dependencies.yml). uv pip install --quiet --system --group plugin # Do not load tests/conftest.py (Django host setup); plugin tests only need pytest + stdlib. -python -m pytest --confcutdir=tests/plugin --override-ini addopts= tests/plugin/test_smoke.py -v +PYTEST_PLUGIN_OPTS="${PYTEST_PLUGIN_OPTS:---timeout=120 --timeout-method=thread --reruns 1 --reruns-delay 5}" +# shellcheck disable=SC2086 +python -m pytest --confcutdir=tests/plugin --override-ini addopts= \ + $PYTEST_PLUGIN_OPTS tests/plugin/test_smoke.py -v diff --git a/tests/plugin/conftest.py b/tests/plugin/conftest.py index 5e74a91..36896b2 100644 --- a/tests/plugin/conftest.py +++ b/tests/plugin/conftest.py @@ -22,6 +22,40 @@ TEST_BRANCH = f"local-{TEST_LANG_CODE}" TEST_VERSION = "test-1.0.0" +# E2E class must run before add-or-update Celery flow in test_functional.py. +_FUNCTIONAL_CLASS_ORDER = ( + "TestQuickBookRoundTrip", + "TestBoostComponentServiceE2E", + "TestAddOrUpdateCeleryFlow", +) + + +def pytest_collection_modifyitems(items: list[pytest.Item]) -> None: + """Enforce functional test class order (pytest default order is not guaranteed).""" + order_index = {name: i for i, name in enumerate(_FUNCTIONAL_CLASS_ORDER)} + + def sort_key(item: pytest.Item) -> tuple[int, int, str]: + cls = getattr(item, "cls", None) + if cls is None: + return (len(_FUNCTIONAL_CLASS_ORDER), 0, item.nodeid) + line = item.location[1] if item.location else 0 + return ( + order_index.get(cls.__name__, len(_FUNCTIONAL_CLASS_ORDER)), + line, + item.nodeid, + ) + + functional = [ + item + for item in items + if item.nodeid.startswith("tests/plugin/test_functional.py") + ] + if not functional: + return + functional.sort(key=sort_key) + others = [item for item in items if item not in functional] + items[:] = others + functional + @pytest.fixture(scope="session") def live_base_url() -> str: @@ -73,4 +107,11 @@ def test_repo(weblate_ssh_pubkey: str) -> EphemeralGitHubRepo: manager.add_deploy_key(weblate_ssh_pubkey) yield manager finally: - manager.delete_repo() + try: + manager.delete_repo() + except Exception as exc: + print( + f"WARNING: failed to delete ephemeral repo {repo_name}: {exc}", + flush=True, + ) + raise diff --git a/tests/plugin/test_functional.py b/tests/plugin/test_functional.py index 185b7c7..7b2a4da 100644 --- a/tests/plugin/test_functional.py +++ b/tests/plugin/test_functional.py @@ -82,9 +82,11 @@ def created_project_component(weblate_api: WeblateAPI) -> CreatedProjectComponen # --------------------------------------------------------------------------- +@pytest.mark.slow class TestQuickBookRoundTrip: """Upload QBK, translate a unit, download translated file.""" + @pytest.mark.timeout(120) def test_units_extracted( self, weblate_api: WeblateAPI, @@ -102,6 +104,7 @@ def test_units_extracted( ] assert any(KNOWN_SOURCE_STRING in s for s in sources), sources[:5] + @pytest.mark.timeout(120) def test_submit_translation( self, weblate_api: WeblateAPI, @@ -120,6 +123,7 @@ def test_submit_translation( unit_url = WeblateAPI.unit_api_url(match) weblate_api.submit_translation(unit_url, ZH_HANS_TRANSLATION) + @pytest.mark.timeout(120) def test_download_translated_qbk( self, weblate_api: WeblateAPI, @@ -140,6 +144,7 @@ def test_download_translated_qbk( # --------------------------------------------------------------------------- +@pytest.mark.slow class TestBoostComponentServiceE2E: """Exercise BoostComponentService inside the Weblate container.""" @@ -215,6 +220,7 @@ def _service_snippet( print(json.dumps(out)) """ + @pytest.mark.timeout(180) def test_clone_and_scan(self, exec_python, test_repo: EphemeralGitHubRepo) -> None: out = json.loads( exec_python(self._service_snippet(test_repo, run_process_all=False)) @@ -224,6 +230,7 @@ def test_clone_and_scan(self, exec_python, test_repo: EphemeralGitHubRepo) -> No assert "quickbook" in formats assert any(c["format"] == "asciidoc" for c in out["configs"]) + @pytest.mark.timeout(180) def test_project_component_creation( self, exec_python, test_repo: EphemeralGitHubRepo ) -> None: @@ -253,6 +260,7 @@ def test_project_component_creation( ) assert check == "ok" + @pytest.mark.timeout(180) def test_idempotency(self, exec_python, test_repo: EphemeralGitHubRepo) -> None: out = json.loads( exec_python( @@ -291,9 +299,11 @@ def add_or_update_task(api_token: str, test_repo: EphemeralGitHubRepo) -> str: return str(data["task_id"]) +@pytest.mark.slow class TestAddOrUpdateCeleryFlow: """POST /boost-endpoint/add-or-update/ and poll Celery completion.""" + @pytest.mark.timeout(360) def test_add_or_update_task_completes( self, weblate_api: WeblateAPI, add_or_update_task: str ) -> None: diff --git a/tests/plugin/test_rate_limit.py b/tests/plugin/test_rate_limit.py index 0a79cd6..57dea27 100644 --- a/tests/plugin/test_rate_limit.py +++ b/tests/plugin/test_rate_limit.py @@ -78,9 +78,11 @@ def rate_limit_api_token() -> str: return docker_exec_python(_RATE_LIMIT_USER_SNIPPET.strip()) +@pytest.mark.slow class TestBoostEndpointRateLimit: """Live-stack rate limit enforcement for Boost endpoint routes.""" + @pytest.mark.timeout(60) def test_info_returns_429_when_rate_limited( self, rate_limit_api_token: str ) -> None: @@ -108,6 +110,7 @@ def test_info_returns_429_when_rate_limited( if get_response_header(last_headers, "X-RateLimit-Limit") is not None: assert int(get_response_header(last_headers, "X-RateLimit-Limit")) == limit + @pytest.mark.timeout(60) def test_add_or_update_returns_429_when_rate_limited( self, rate_limit_api_token: str ) -> None: diff --git a/uv.lock b/uv.lock index 8611d1c..f9e0c3b 100644 --- a/uv.lock +++ b/uv.lock @@ -714,6 +714,7 @@ lint = [ ] plugin = [ {name = "pytest"}, + {name = "pytest-rerunfailures"}, {name = "pytest-timeout"} ] pre-commit = [ @@ -746,6 +747,7 @@ lint = [ ] plugin = [ {name = "pytest", specifier = "==9.0.3"}, + {name = "pytest-rerunfailures", specifier = "==16.3"}, {name = "pytest-timeout", specifier = "==2.4.0"} ] pre-commit = [ @@ -2458,6 +2460,19 @@ wheels = [ {url = "https://files.pythonhosted.org/packages/9d/7a/d968e294073affff457b041c2be9868a40c1c71f4a35fcc1e45e5493067b/pytest_cov-7.1.0-py3-none-any.whl", hash = "sha256:a0461110b7865f9a271aa1b51e516c9a95de9d696734a2f71e3e78f46e1d4678", size = 22876, upload-time = "2026-03-21T20:11:14.438Z"} ] +[[package]] +dependencies = [ + {name = "packaging"}, + {name = "pytest"} +] +name = "pytest-rerunfailures" +sdist = {url = "https://files.pythonhosted.org/packages/4d/f0/74f8e685be7ecd1572c1256132f18fce3a665d7e07649a3f23b7eb2d3bec/pytest_rerunfailures-16.3.tar.gz", hash = "sha256:37c9b1231c8083e9f4e724f50f7a21241822f9516c15c700ebbf218d6452355c", size = 34148, upload-time = "2026-05-22T06:51:22.292Z"} +source = {registry = "https://pypi.org/simple"} +version = "16.3" +wheels = [ + {url = "https://files.pythonhosted.org/packages/f8/98/58a71d68d3126d7f6a6ed1944c37ec207a4ff3dc66cad3bed7b59d38df61/pytest_rerunfailures-16.3-py3-none-any.whl", hash = "sha256:6bdfb8ffb46c46072e6c16bdedee38b6c13eac620d9415ed5b63152cbf283170", size = 15396, upload-time = "2026-05-22T06:51:20.547Z"} +] + [[package]] dependencies = [ {name = "pytest"}