plugin test reliability and stabilization

.github/README.md

@@ -27,6 +27,35 @@ GitHub Actions and CI/CD helpers for this repository.
 
 Callable workflows (`ci-*`, `ci-plugin-*`) are triggered only via `workflow_call` from `ci.yml`, not directly on push.
 
+## Plugin integration jobs
+
+Three callable workflows exercise the live Weblate Docker stack ([`docker/docker-compose.ci.yml`](../docker/docker-compose.ci.yml)). Each job builds the image, runs `compose up -d --wait`, probes `/healthz/` and the Boost ping endpoint, creates an API token (with retry), then runs pytest with `pytest-timeout` and one rerun on failure.
+
+| Job | Workflow | Typical duration | Hard limit (`timeout-minutes`) | Notes |
+|-----|----------|------------------|--------------------------------|-------|
+| Plugin smoke | [`ci-plugin-smoke.yml`](workflows/ci-plugin-smoke.yml) | ~8–12 min | 15 | Stack image build dominates |
+| Plugin functional | [`ci-plugin-functional.yml`](workflows/ci-plugin-functional.yml) | ~15–22 min | 25 | GitHub E2E needs repository secret |
+| Plugin auth | [`ci-plugin-auth.yml`](workflows/ci-plugin-auth.yml) | ~8–12 min | 10 | Auth + rate-limit tests |
+
+### Secrets and environment
+
+| Variable | Where | Purpose |
+|----------|-------|---------|
+| `GH_TEST_REPO_TOKEN` | Repository secret (functional job only) | Classic PAT with `repo` scope for ephemeral GitHub repos in [`tests/plugin/test_functional.py`](../tests/plugin/test_functional.py). If unset, GitHub/Celery E2E tests are skipped. |
+| `HEALTH_TIMEOUT` | CI workflow env / shell default | Seconds to wait for `/healthz/` after compose `--wait`. Defaults: smoke/auth **240**, functional **300**. |
+| `PYTEST_PLUGIN_OPTS` | Optional override in entrypoint scripts | Default includes `--timeout`, `--timeout-method=thread`, `--reruns 1`, `--reruns-delay 5`. Smoke/auth use `--timeout=120`; functional uses `--timeout=300`. |
+| `WEBLATE_PORT` | Optional | Host port for Weblate (default **8080**). |
+
+### Local reproduction
+
+```bash
+bash scripts/plugin-smoke.sh
+bash scripts/plugin-auth.sh
+GH_TEST_REPO_TOKEN=<classic PAT with repo> bash scripts/plugin-functional.sh
+```
+
+Skip slow plugin tests during local iteration: add `-m "not slow"` to the pytest invocation in the script, or set `PYTEST_PLUGIN_OPTS` accordingly.
+
 ## Other paths
 
 | Path | Role |

.github/workflows/ci-plugin-auth.yml

@@ -33,6 +33,8 @@ jobs:
         version: 0.11.12
 
     - name: Run plugin auth tests
+      env:
+        HEALTH_TIMEOUT: '240'
       run: bash scripts/plugin-auth.sh
 
     - name: Upload logs on failure

.github/workflows/ci-plugin-functional.yml

@@ -45,6 +45,7 @@ jobs:
     - name: Run plugin functional tests
       env:
         GH_TEST_REPO_TOKEN: ${{ secrets.GH_TEST_REPO_TOKEN }}
+        HEALTH_TIMEOUT: '300'
       run: bash scripts/plugin-functional.sh
 
     - name: Upload logs on failure

.github/workflows/ci-plugin-smoke.yml

@@ -33,6 +33,8 @@ jobs:
         version: 0.11.12
 
     - name: Run plugin smoke tests
+      env:
+        HEALTH_TIMEOUT: '240'
       run: bash scripts/plugin-smoke.sh
 
     - name: Upload logs on failure

pyproject.toml

@@ -16,6 +16,7 @@ lint = [
 ]
 plugin = [
   "pytest==9.0.3",
+  "pytest-rerunfailures==16.3",
   "pytest-timeout==2.4.0"
 ]
 pre-commit = [
@@ -132,7 +133,8 @@ unauthorized_licenses = []
 [tool.pytest.ini_options]
 addopts = ["-m", "not plugin"]
 markers = [
-  "plugin: requires live Weblate stack (Docker Compose) and optional WEBLATE_API_TOKEN"
+  "plugin: requires live Weblate stack (Docker Compose) and optional WEBLATE_API_TOKEN",
+  "slow: long-running plugin integration test"
 ]
 python_classes = ["Test*"]
 python_files = ["test_*.py", "*_test.py"]

scripts/README.md

@@ -24,5 +24,8 @@ bash scripts/plugin-smoke.sh
 source scripts/lib/weblate-stack.sh
 stack_build
 stack_up
-stack_wait_healthy 120
+stack_wait_healthy 240
+stack_wait_api_ready
 ```
+
+CI entrypoints use `compose up -d --wait`, `HEALTH_TIMEOUT` (smoke/auth **240**, functional **300**), `stack_create_token_retry`, and `PYTEST_PLUGIN_OPTS` for pytest timeout/reruns. See [`.github/README.md`](../.github/README.md#plugin-integration-jobs).

scripts/lib/weblate-stack.sh

@@ -16,11 +16,12 @@ stack_build() {
 }
 
 stack_up() {
-    compose up -d "$@"
+    # Wait for compose healthchecks (Postgres, Redis, Weblate) before returning.
+    compose up -d --wait "$@"
 }
 
 stack_wait_healthy() {
-    local timeout="${1:-120}"
+    local timeout="${1:-240}"
     local port="${WEBLATE_PORT:-8080}"
     local url="http://localhost:${port}/healthz/"
     local interval=5
@@ -42,6 +43,27 @@ stack_wait_healthy() {
     return 1
 }
 
+stack_wait_api_ready() {
+    local port="${WEBLATE_PORT:-8080}"
+    local url="http://localhost:${port}/boost-endpoint/plugin-ping/"
+    local attempts="${1:-12}"
+    local interval="${2:-5}"
+    local i=0
+
+    echo "Waiting for Boost endpoint at ${url}..."
+    while [ "$i" -lt "$attempts" ]; do
+        if curl -sf "$url" > /dev/null 2>&1; then
+            echo "Boost endpoint is ready (after $((i * interval))s)."
+            return 0
+        fi
+        sleep "$interval"
+        i=$((i + 1))
+    done
+
+    echo "ERROR: Boost endpoint did not become ready in $((attempts * interval))s."
+    return 1
+}
+
 stack_ensure_github_known_hosts() {
     # Weblate git uses GIT_SSH with UserKnownHostsFile=/app/data/ssh/known_hosts
     # and StrictHostKeyChecking=yes. Celery/component sync over git@github.com fails
@@ -79,6 +101,29 @@ t = Token.objects.create(user=u, key=get_token("wlp" if u.is_bot else "wlu"))
 print(t.key)'
 }
 
+stack_create_token_retry() {
+    local user="${1:-admin}"
+    local attempts="${2:-3}"
+    local interval="${3:-5}"
+    local attempt=1
+    local token=""
+
+    while [ "$attempt" -le "$attempts" ]; do
+        if token="$(stack_create_token "$user")" && [ -n "$token" ]; then
+            echo "$token"
+            return 0
+        fi
+        if [ "$attempt" -lt "$attempts" ]; then
+            echo "Token creation attempt ${attempt}/${attempts} failed; retrying in ${interval}s..." >&2
+            sleep "$interval"
+        fi
+        attempt=$((attempt + 1))
+    done
+
+    echo "ERROR: Failed to create API token for user ${user} after ${attempts} attempts." >&2
+    return 1
+}
+
 stack_logs() {
     local file="${1:-}"
     if [ -n "$file" ]; then

scripts/plugin-auth.sh

@@ -30,10 +30,11 @@ echo "=== Starting stack ==="
 stack_up
 
 echo "=== Waiting for Weblate ==="
-stack_wait_healthy "${HEALTH_TIMEOUT:-120}"
+stack_wait_healthy "${HEALTH_TIMEOUT:-240}"
+stack_wait_api_ready
 
 echo "=== Creating API token ==="
-WEBLATE_API_TOKEN="$(stack_create_token admin)"
+WEBLATE_API_TOKEN="$(stack_create_token_retry admin)"
 export WEBLATE_API_TOKEN
 export WEBLATE_LIVE_BASE_URL="${WEBLATE_LIVE_BASE_URL:-http://localhost:${WEBLATE_PORT:-8080}}"
 export WEBLATE_COMPOSE_FILE="${COMPOSE_FILE}"
@@ -43,5 +44,8 @@ export BOOST_ENDPOINT_THROTTLE_ADD_OR_UPDATE="${BOOST_ENDPOINT_THROTTLE_ADD_OR_U
 
 echo "=== Running auth tests ==="
 uv pip install --quiet --system --group plugin
+PYTEST_PLUGIN_OPTS="${PYTEST_PLUGIN_OPTS:---timeout=120 --timeout-method=thread --reruns 1 --reruns-delay 5}"
+# shellcheck disable=SC2086
 python -m pytest --confcutdir=tests/plugin --override-ini addopts= \
+    $PYTEST_PLUGIN_OPTS \
     tests/plugin/test_auth.py tests/plugin/test_rate_limit.py -v

scripts/plugin-functional.sh

@@ -30,10 +30,11 @@ echo "=== Starting stack ==="
 stack_up
 
 echo "=== Waiting for Weblate ==="
-stack_wait_healthy "${HEALTH_TIMEOUT:-180}"
+stack_wait_healthy "${HEALTH_TIMEOUT:-300}"
+stack_wait_api_ready
 
 echo "=== Creating API token ==="
-WEBLATE_API_TOKEN="$(stack_create_token admin)"
+WEBLATE_API_TOKEN="$(stack_create_token_retry admin)"
 export WEBLATE_API_TOKEN
 export WEBLATE_LIVE_BASE_URL="${WEBLATE_LIVE_BASE_URL:-http://localhost:${WEBLATE_PORT:-8080}}"
 export WEBLATE_COMPOSE_FILE="${COMPOSE_FILE}"
@@ -59,5 +60,7 @@ fi
 
 echo "=== Running functional tests ==="
 uv pip install --quiet --system --group plugin
+PYTEST_PLUGIN_OPTS="${PYTEST_PLUGIN_OPTS:---timeout=300 --timeout-method=thread --reruns 1 --reruns-delay 5}"
+# shellcheck disable=SC2086
 python -m pytest --confcutdir=tests/plugin --override-ini addopts= \
-    tests/plugin/test_functional.py -v --timeout=300
+    $PYTEST_PLUGIN_OPTS tests/plugin/test_functional.py -v

scripts/plugin-smoke.sh

@@ -30,10 +30,11 @@ echo "=== Starting stack ==="
 stack_up
 
 echo "=== Waiting for Weblate ==="
-stack_wait_healthy "${HEALTH_TIMEOUT:-120}"
+stack_wait_healthy "${HEALTH_TIMEOUT:-240}"
+stack_wait_api_ready
 
 echo "=== Creating API token ==="
-WEBLATE_API_TOKEN="$(stack_create_token admin)"
+WEBLATE_API_TOKEN="$(stack_create_token_retry admin)"
 export WEBLATE_API_TOKEN
 export WEBLATE_LIVE_BASE_URL="${WEBLATE_LIVE_BASE_URL:-http://localhost:${WEBLATE_PORT:-8080}}"
 export WEBLATE_COMPOSE_FILE="${COMPOSE_FILE}"
@@ -44,4 +45,7 @@ echo "=== Running smoke tests ==="
 # --system: setup-python on CI has no project venv (matches ci-dependencies.yml).
 uv pip install --quiet --system --group plugin
 # Do not load tests/conftest.py (Django host setup); plugin tests only need pytest + stdlib.
-python -m pytest --confcutdir=tests/plugin --override-ini addopts= tests/plugin/test_smoke.py -v
+PYTEST_PLUGIN_OPTS="${PYTEST_PLUGIN_OPTS:---timeout=120 --timeout-method=thread --reruns 1 --reruns-delay 5}"
+# shellcheck disable=SC2086
+python -m pytest --confcutdir=tests/plugin --override-ini addopts= \
+    $PYTEST_PLUGIN_OPTS tests/plugin/test_smoke.py -v

tests/plugin/conftest.py

@@ -22,6 +22,40 @@
 TEST_BRANCH = f"local-{TEST_LANG_CODE}"
 TEST_VERSION = "test-1.0.0"
 
+# E2E class must run before add-or-update Celery flow in test_functional.py.
+_FUNCTIONAL_CLASS_ORDER = (
+    "TestQuickBookRoundTrip",
+    "TestBoostComponentServiceE2E",
+    "TestAddOrUpdateCeleryFlow",
+)
+
+
+def pytest_collection_modifyitems(items: list[pytest.Item]) -> None:
+    """Enforce functional test class order (pytest default order is not guaranteed)."""
+    order_index = {name: i for i, name in enumerate(_FUNCTIONAL_CLASS_ORDER)}
+
+    def sort_key(item: pytest.Item) -> tuple[int, int, str]:
+        cls = getattr(item, "cls", None)
+        if cls is None:
+            return (len(_FUNCTIONAL_CLASS_ORDER), 0, item.nodeid)
+        line = item.location[1] if item.location else 0
+        return (
+            order_index.get(cls.__name__, len(_FUNCTIONAL_CLASS_ORDER)),
+            line,
+            item.nodeid,
+        )
+
+    functional = [
+        item
+        for item in items
+        if item.nodeid.startswith("tests/plugin/test_functional.py")
+    ]
+    if not functional:
+        return
+    functional.sort(key=sort_key)
+    others = [item for item in items if item not in functional]
+    items[:] = others + functional
+
 
 @pytest.fixture(scope="session")
 def live_base_url() -> str:
@@ -73,4 +107,11 @@ def test_repo(weblate_ssh_pubkey: str) -> EphemeralGitHubRepo:
         manager.add_deploy_key(weblate_ssh_pubkey)
         yield manager
     finally:
-        manager.delete_repo()
+        try:
+            manager.delete_repo()
+        except Exception as exc:
+            print(
+                f"WARNING: failed to delete ephemeral repo {repo_name}: {exc}",
+                flush=True,
+            )
+            raise

tests/plugin/test_functional.py

@@ -82,9 +82,11 @@ def created_project_component(weblate_api: WeblateAPI) -> CreatedProjectComponen
 # ---------------------------------------------------------------------------
 
 
+@pytest.mark.slow
 class TestQuickBookRoundTrip:
     """Upload QBK, translate a unit, download translated file."""
 
+    @pytest.mark.timeout(120)
     def test_units_extracted(
         self,
         weblate_api: WeblateAPI,
@@ -102,6 +104,7 @@ def test_units_extracted(
         ]
         assert any(KNOWN_SOURCE_STRING in s for s in sources), sources[:5]
 
+    @pytest.mark.timeout(120)
     def test_submit_translation(
         self,
         weblate_api: WeblateAPI,
@@ -120,6 +123,7 @@ def test_submit_translation(
         unit_url = WeblateAPI.unit_api_url(match)
         weblate_api.submit_translation(unit_url, ZH_HANS_TRANSLATION)
 
+    @pytest.mark.timeout(120)
     def test_download_translated_qbk(
         self,
         weblate_api: WeblateAPI,
@@ -140,6 +144,7 @@ def test_download_translated_qbk(
 # ---------------------------------------------------------------------------
 
 
+@pytest.mark.slow
 class TestBoostComponentServiceE2E:
     """Exercise BoostComponentService inside the Weblate container."""
 
@@ -215,6 +220,7 @@ def _service_snippet(
 print(json.dumps(out))
 """
 
+    @pytest.mark.timeout(180)
     def test_clone_and_scan(self, exec_python, test_repo: EphemeralGitHubRepo) -> None:
         out = json.loads(
             exec_python(self._service_snippet(test_repo, run_process_all=False))
@@ -224,6 +230,7 @@ def test_clone_and_scan(self, exec_python, test_repo: EphemeralGitHubRepo) -> No
         assert "quickbook" in formats
         assert any(c["format"] == "asciidoc" for c in out["configs"])
 
+    @pytest.mark.timeout(180)
     def test_project_component_creation(
         self, exec_python, test_repo: EphemeralGitHubRepo
     ) -> None:
@@ -253,6 +260,7 @@ def test_project_component_creation(
         )
         assert check == "ok"
 
+    @pytest.mark.timeout(180)
     def test_idempotency(self, exec_python, test_repo: EphemeralGitHubRepo) -> None:
         out = json.loads(
             exec_python(
@@ -291,9 +299,11 @@ def add_or_update_task(api_token: str, test_repo: EphemeralGitHubRepo) -> str:
     return str(data["task_id"])
 
 
+@pytest.mark.slow
 class TestAddOrUpdateCeleryFlow:
     """POST /boost-endpoint/add-or-update/ and poll Celery completion."""
 
+    @pytest.mark.timeout(360)
     def test_add_or_update_task_completes(
         self, weblate_api: WeblateAPI, add_or_update_task: str
     ) -> None:

tests/plugin/test_rate_limit.py

@@ -78,9 +78,11 @@ def rate_limit_api_token() -> str:
     return docker_exec_python(_RATE_LIMIT_USER_SNIPPET.strip())
 
 
+@pytest.mark.slow
 class TestBoostEndpointRateLimit:
     """Live-stack rate limit enforcement for Boost endpoint routes."""
 
+    @pytest.mark.timeout(60)
     def test_info_returns_429_when_rate_limited(
         self, rate_limit_api_token: str
     ) -> None:
@@ -108,6 +110,7 @@ def test_info_returns_429_when_rate_limited(
         if get_response_header(last_headers, "X-RateLimit-Limit") is not None:
             assert int(get_response_header(last_headers, "X-RateLimit-Limit")) == limit
 
+    @pytest.mark.timeout(60)
     def test_add_or_update_returns_429_when_rate_limited(
         self, rate_limit_api_token: str
     ) -> None: