diff --git a/.env.example b/.env.example index 9a259b9..240da4e 100644 --- a/.env.example +++ b/.env.example @@ -20,6 +20,10 @@ # Host port bound to 127.0.0.1; nginx proxies HTTPS /weblate/ to this port. WEBLATE_PORT=8080 +# External Docker network where shared Redis runs (required; compose fails if unset). +# List networks after starting boost-data-collector: docker network ls +REDIS_EXTERNAL_NETWORK=shared-redis-network + # --------------------------------------------------------------------------- # Required secrets (compose fails if unset) # --------------------------------------------------------------------------- @@ -36,7 +40,8 @@ WEBLATE_LOGLEVEL=INFO WEBLATE_SITE_TITLE=Example Weblate WEBLATE_ADMIN_NAME=Weblate Admin WEBLATE_ADMIN_EMAIL=admin@example.com -# Required. Hostname users see (no scheme); match your public URL host. +# Required in .env (loaded via env_file). Hostname users see (no scheme); must match +# WEBLATE_ALLOWED_HOSTS and your public nginx hostname. Replace example.com placeholders. WEBLATE_SITE_DOMAIN=weblate.example.com WEBLATE_SERVER_EMAIL=noreply@example.com WEBLATE_DEFAULT_FROM_EMAIL=noreply@example.com @@ -76,48 +81,51 @@ WEBLATE_SECURE_PROXY_SSL_HEADER=HTTP_X_FORWARDED_PROTO,https # WEBLATE_AUTH_LDAP_USER_ATTR_MAP=first_name:name,email:mail # --------------------------------------------------------------------------- -# PostgreSQL (host; POSTGRES_HOST also set in docker-compose.cd.yml) +# PostgreSQL (host) # --------------------------------------------------------------------------- +# CD compose pins POSTGRES_HOST and POSTGRES_PORT in docker-compose.cd.yml +# (host.docker.internal, default 5432). Values below apply via env_file for user/db. POSTGRES_USER=weblate_app # Canonical name per Weblate Docker docs (POSTGRES_DATABASE is an alias some images accept). POSTGRES_DB=weblate_db POSTGRES_DATABASE=weblate_db -POSTGRES_HOST=host.docker.internal -POSTGRES_PORT=5432 +# Ignored in CD (compose pins host/port). Documented for reference / non-CD stacks. +# POSTGRES_HOST=host.docker.internal +# POSTGRES_PORT=5432 # Optional: POSTGRES_PASSWORD_FILE=/run/secrets/db_password -# --------------------------------------------------------------------------- -# Redis (shared boost-data-collector stack; host/port in docker-compose.cd.yml) -# --------------------------------------------------------------------------- - # Logical DB 1 avoids clashing with other apps on the same Redis (default for Weblate is 1). REDIS_DB=1 -# Override only if you change the external network or service name: +# Reference only for CI / local overrides (not used by docker-compose.cd.yml): # REDIS_HOST=redis # REDIS_PORT=6379 # --------------------------------------------------------------------------- -# Mail server +# Mail server (production: required for notifications; env_file only) # --------------------------------------------------------------------------- +# Set host, user, and password for your SMTP provider. For staging without mail, +# uncomment WEBLATE_EMAIL_BACKEND dummy backend below instead of real SMTP. WEBLATE_EMAIL_HOST=smtp.example.com WEBLATE_EMAIL_PORT=587 -WEBLATE_EMAIL_HOST_USER= -WEBLATE_EMAIL_HOST_PASSWORD= +WEBLATE_EMAIL_HOST_USER=replace-with-smtp-user +WEBLATE_EMAIL_HOST_PASSWORD=replace-with-smtp-password WEBLATE_EMAIL_USE_TLS=1 WEBLATE_EMAIL_USE_SSL=0 # WEBLATE_EMAIL_BACKEND=django.core.mail.backends.dummy.EmailBackend # --------------------------------------------------------------------------- -# GitHub (VCS push / PR; must use WEBLATE_ prefix per Weblate docs) +# GitHub (production: required for VCS and add-or-update; env_file only) # --------------------------------------------------------------------------- +# PAT needs repo scope for clone/push. Loaded via env_file; not validated at compose up. +# See pre-deploy checklist in docs/deployment-runbook.md for token rotation. WEBLATE_GITHUB_HOST=api.github.com -WEBLATE_GITHUB_USERNAME= -WEBLATE_GITHUB_TOKEN= +WEBLATE_GITHUB_USERNAME=replace-with-github-username +WEBLATE_GITHUB_TOKEN=replace-with-github-pat # WEBLATE_GITLAB_USERNAME= # WEBLATE_GITLAB_HOST= @@ -149,7 +157,19 @@ WEBLATE_GITHUB_TOKEN= CLIENT_MAX_BODY_SIZE=1000M # --------------------------------------------------------------------------- -# Celery (set in docker-compose.cd.yml; override if needed) +# Celery (Weblate worker process count; loaded via env_file) # --------------------------------------------------------------------------- +# Number of Celery worker processes in the Weblate container (not a separate broker setting). +# Default 1 for light staging; increase to 2-4 when add-or-update tasks queue up. +# Restart the stack after changing. CELERY_SINGLE_PROCESS=1 + +# --------------------------------------------------------------------------- +# Boost endpoint plugin (production rate limits) +# --------------------------------------------------------------------------- +# Read at container boot by settings_override.py (DRF scoped throttles). +# Format: N/second|minute|hour|day + +BOOST_ENDPOINT_THROTTLE_INFO=60/minute +BOOST_ENDPOINT_THROTTLE_ADD_OR_UPDATE=10/hour diff --git a/.github/README.md b/.github/WORKFLOWS.md similarity index 98% rename from .github/README.md rename to .github/WORKFLOWS.md index 8fa151a..f12cce2 100644 --- a/.github/README.md +++ b/.github/WORKFLOWS.md @@ -4,9 +4,9 @@ SPDX-FileCopyrightText: 2026 William Jin SPDX-License-Identifier: BSL-1.0 --> -# `.github/` +# CI/CD workflows -GitHub Actions and CI/CD helpers for this repository. +GitHub Actions and CI/CD helpers for this repository (see [`.github/`](../.github/) for workflow YAML). ## Workflows diff --git a/README.md b/README.md index 997d760..75d86ac 100644 --- a/README.md +++ b/README.md @@ -296,7 +296,7 @@ Triggered on push and PR to `main` and `develop`. Calls eight reusable sub-workf All `ci-plugin-*` jobs build the CI Docker stack (`docker/docker-compose.ci.yml`), wait for the healthcheck, create an API token, run the corresponding pytest suite under `tests/plugin/`, and tear down. -[`weblate-pin-bump.yml`](.github/workflows/weblate-pin-bump.yml) runs on a schedule (Monday 09:00 UTC) and opens a PR when a newer PyPI Weblate release has a matching Docker fixed tag. See [`.github/README.md`](.github/README.md#weblate-version-pinning). +[`weblate-pin-bump.yml`](.github/workflows/weblate-pin-bump.yml) runs on a schedule (Monday 09:00 UTC) and opens a PR when a newer PyPI Weblate release has a matching Docker fixed tag. See [`.github/WORKFLOWS.md`](.github/WORKFLOWS.md#weblate-version-pinning). ### CD (`cd.yml`) and production promotion (`promote-main.yml`) @@ -349,7 +349,7 @@ Each script builds `docker/docker-compose.ci.yml`, waits for health, runs its py | API reference | [`docs/boost-endpoint-api.md`](docs/boost-endpoint-api.md) | Full request/response docs for the Boost endpoint | | Route registration | [`docs/plugin-http-routes.md`](docs/plugin-http-routes.md) | How and why routes are registered at startup | | Docker files | [`docker/README.md`](docker/README.md) | Dockerfile and Compose usage for CI and CD | -| CI/CD workflows | [`.github/README.md`](.github/README.md) | Workflow index, staging/production secrets, `PROMOTE_PAT` | +| CI/CD workflows | [`.github/WORKFLOWS.md`](.github/WORKFLOWS.md) | Workflow index, staging/production secrets, `PROMOTE_PAT` | ## Contributing diff --git a/docker/README.md b/docker/README.md index e619669..3d9bb0b 100644 --- a/docker/README.md +++ b/docker/README.md @@ -21,6 +21,7 @@ docker compose -f docker/docker-compose.ci.yml up -d # CD on deploy server (copy .env.example to repo-root .env; set WEBLATE_URL_PREFIX, REDIS_DB, secrets): cp .env.example .env +# set WEBLATE_URL_PREFIX, REDIS_DB, secrets and all needed values in .env file. docker compose -f docker/docker-compose.cd.yml --env-file .env build docker compose -f docker/docker-compose.cd.yml --env-file .env up -d ``` diff --git a/docker/docker-compose.cd.yml b/docker/docker-compose.cd.yml index c7f6304..b498e08 100644 --- a/docker/docker-compose.cd.yml +++ b/docker/docker-compose.cd.yml @@ -10,6 +10,7 @@ services: build: context: .. dockerfile: docker/Dockerfile.weblate-plugin + # Operator config: copy .env.example to repo-root .env (see docs/deployment-runbook.md). env_file: - ../.env ports: @@ -17,17 +18,12 @@ services: extra_hosts: - host.docker.internal:host-gateway environment: - WEBLATE_SITE_DOMAIN: ${WEBLATE_SITE_DOMAIN:-weblate.example.com} + POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:?set in .env} WEBLATE_ADMIN_PASSWORD: ${WEBLATE_ADMIN_PASSWORD:?set in .env} - WEBLATE_DEBUG: ${WEBLATE_DEBUG:-0} POSTGRES_HOST: host.docker.internal - POSTGRES_PORT: '5432' - POSTGRES_USER: ${POSTGRES_USER:-weblate_app} - POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:?set in .env} - POSTGRES_DATABASE: ${POSTGRES_DATABASE:-weblate_db} + POSTGRES_PORT: ${POSTGRES_PORT:-5432} REDIS_HOST: redis - REDIS_PORT: '6379' - CELERY_SINGLE_PROCESS: ${CELERY_SINGLE_PROCESS:-1} + REDIS_PORT: ${REDIS_PORT:-6379} healthcheck: test: [CMD, curl, -sf, 'http://localhost:8080${WEBLATE_URL_PREFIX:-}/healthz/'] interval: 10s @@ -43,7 +39,8 @@ services: networks: - bdc_redis +# Join external Redis network (REDIS_EXTERNAL_NETWORK in .env). networks: bdc_redis: external: true - name: boost-data-collector_default + name: ${REDIS_EXTERNAL_NETWORK:?set in .env} diff --git a/docs/deployment-runbook.md b/docs/deployment-runbook.md index a95c3fd..e3b058a 100644 --- a/docs/deployment-runbook.md +++ b/docs/deployment-runbook.md @@ -14,7 +14,7 @@ Step-by-step guide for deploying `cppa-weblate-plugin` to a staging or productio |-------------|---------| | Docker Engine | 24 + with Compose v2 (`docker compose`) | | Host PostgreSQL | 16 recommended; a dedicated user and database (see [Database setup](#database-setup)) | -| Redis | 7+; shared via the `boost-data-collector_default` external Docker network | +| Redis | 7+; shared via external Docker network (`REDIS_EXTERNAL_NETWORK` in `.env`, required at `compose up`) | | Reverse proxy | nginx (or equivalent) terminating TLS and proxying to `127.0.0.1:8080` | | Git checkout | Repository cloned to `/opt/cppa-weblate-plugin` on the deploy server | @@ -31,12 +31,14 @@ Ensure `pg_hba.conf` allows connections from the Docker bridge network (`172.17. ## Environment File -Copy `.env.example` to the repo root as `.env` and fill in every value marked `replace-*`: +Copy `.env.example` to the repo root as `.env` and fill in every value marked `replace-*` (including SMTP and GitHub credentials), and replace all `example.com` placeholders with your real hostname: ```bash cp .env.example .env ``` +Before the first deploy or any production upgrade, complete the [Pre-Deploy Checklist](#pre-deploy-checklist). + ### Required secrets | Variable | Purpose | @@ -44,34 +46,115 @@ cp .env.example .env | `POSTGRES_PASSWORD` | Host Postgres password for `weblate_app` | | `WEBLATE_ADMIN_PASSWORD` | Initial admin account password | -Compose refuses to start if either is unset (enforced by `${VAR:?set in .env}` syntax in `docker-compose.cd.yml`). +Compose refuses to start if either is unset (enforced by `${VAR:?set in .env}` in `docker-compose.cd.yml` `environment:`). + +### Production integration (`.env` only; fill before go-live) + +Weblate does not fail `compose up` if these are missing, but production needs them for real use: + +| Variable | Purpose | +|----------|---------| +| `WEBLATE_EMAIL_HOST`, `WEBLATE_EMAIL_HOST_USER`, `WEBLATE_EMAIL_HOST_PASSWORD` | Outbound mail (notifications, password reset). Use dummy `WEBLATE_EMAIL_BACKEND` only on staging without SMTP | +| `WEBLATE_GITHUB_USERNAME`, `WEBLATE_GITHUB_TOKEN` | GitHub API and git operations; **required** for `POST /boost-endpoint/add-or-update/` Celery tasks (clone/push) | + +Rotate `WEBLATE_EMAIL_HOST_PASSWORD` and `WEBLATE_GITHUB_TOKEN` per the [Pre-Deploy Checklist](#pre-deploy-checklist). + +### Compose vs `.env` + +Docker Compose loads operator config from `env_file: ../.env`. The `environment:` block in `docker-compose.cd.yml` only sets: + +| Source | Variables | Purpose | +|--------|-----------|---------| +| **`environment:` fail-fast** | `POSTGRES_PASSWORD`, `WEBLATE_ADMIN_PASSWORD` | Refuse `compose up` if secrets are missing | +| **`environment:` pins** | `POSTGRES_HOST`, `POSTGRES_PORT`, `REDIS_HOST`, `REDIS_PORT` | CD topology; overrides `.env` for these keys | +| **`env_file` only** | All other keys in `.env.example` | Weblate, mail, GitHub, plugin throttles, `CELERY_SINGLE_PROCESS`, etc. | +| **Compose-only (`.env`, not in container)** | `REDIS_EXTERNAL_NETWORK` | External network name Weblate joins (`:?` at compose up; must match `docker network ls` after BDC starts) | + +Do not duplicate pass-through vars in `environment:`; configure them once in `.env`. Set `REDIS_EXTERNAL_NETWORK` to the network that hosts Redis; only `REDIS_DB` tunes Redis logic inside the shared instance. ### Plugin-specific settings -The plugin itself has **no dedicated env vars**. All wiring happens inside the Docker image at build time: +Build-time wiring (no env vars): 1. **`settings_override.py`** is copied to `/app/data/settings-override.py` by the Dockerfile. Weblate's Docker entrypoint `exec()`s this file during settings load. 2. **`WEBLATE_FORMATS`** — the override reads upstream `FormatsConf.FORMATS` via regex, appends `boost_weblate.formats.quickbook.QuickBookFormat`, and writes the result back to `WEBLATE_FORMATS`. No env var needed. 3. **`INSTALLED_APPS`** — the override appends `boost_weblate.endpoint.apps.BoostEndpointConfig`. The app's `ready()` hook then registers `/boost-endpoint/` routes on `weblate.urls.real_patterns`. +Runtime plugin env vars (set in `.env`, read by `settings_override.py` at boot): + +| Variable | Production default | Notes | +|----------|-------------------|-------| +| `BOOST_ENDPOINT_THROTTLE_INFO` | `60/minute` | Scoped rate for `GET /boost-endpoint/info/` | +| `BOOST_ENDPOINT_THROTTLE_ADD_OR_UPDATE` | `10/hour` | Scoped rate for `POST /boost-endpoint/add-or-update/` | + ### Weblate environment variables -Key variables set in the Compose file or `.env` (full reference in `.env.example`): - -| Variable | Default | Notes | -|----------|---------|-------| -| `WEBLATE_PORT` | `8080` | Host port bound to `127.0.0.1`; nginx proxies to this | -| `WEBLATE_SITE_DOMAIN` | `weblate.example.com` | Public hostname (no scheme) | -| `WEBLATE_URL_PREFIX` | `/weblate` | Subpath when behind nginx at `https:///weblate/` | -| `WEBLATE_DEBUG` | `0` | Set `1` only for troubleshooting | -| `WEBLATE_ENABLE_HTTPS` | `1` | Required when TLS terminates at nginx | -| `WEBLATE_IP_PROXY_HEADER` | `HTTP_X_FORWARDED_FOR` | Proxy header for real client IP | -| `POSTGRES_HOST` | `host.docker.internal` | Reaches host Postgres via Docker gateway | -| `POSTGRES_USER` | `weblate_app` | Must match the SQL `CREATE USER` above | -| `POSTGRES_DATABASE` | `weblate_db` | Must match `CREATE DATABASE` above | -| `REDIS_HOST` | `redis` | Resolved via the external `bdc_redis` network | -| `REDIS_DB` | `1` | Logical DB to avoid clashing with other apps on shared Redis | -| `CELERY_SINGLE_PROCESS` | `1` | Single Celery worker process; increase for heavier workloads | +Key variables (full reference in `.env.example`): + +| Variable | Default | Set via | Notes | +|----------|---------|---------|-------| +| `WEBLATE_PORT` | `8080` | `.env` (compose interpolation) | Host port bound to `127.0.0.1`; nginx proxies to this | +| `REDIS_EXTERNAL_NETWORK` | — | `.env` (compose `:?`) | **Required.** External Docker network for shared Redis (set to your BDC network name) | +| `WEBLATE_SITE_DOMAIN` | — | `.env` | **Required.** Public hostname (no scheme); must match `WEBLATE_ALLOWED_HOSTS` | +| `WEBLATE_URL_PREFIX` | `/weblate` | `.env` | Subpath when behind nginx at `https:///weblate/` | +| `WEBLATE_DEBUG` | `0` | `.env` | Set `1` only for troubleshooting | +| `WEBLATE_ENABLE_HTTPS` | `1` | `.env` | Required when TLS terminates at nginx | +| `WEBLATE_IP_PROXY_HEADER` | `HTTP_X_FORWARDED_FOR` | `.env` | Proxy header for real client IP | +| `POSTGRES_HOST` | `host.docker.internal` | **Compose pin** | Not operator-configurable in CD | +| `POSTGRES_PORT` | `5432` | **Compose pin** (`:-5432`) | Override in `.env` only if host Postgres uses a non-default port | +| `POSTGRES_USER` | `weblate_app` | `.env` | Must match the SQL `CREATE USER` above | +| `POSTGRES_DATABASE` | `weblate_db` | `.env` | Must match `CREATE DATABASE` above | +| `REDIS_HOST` | `redis` | **Compose pin** | Service name on external `bdc_redis` network | +| `REDIS_PORT` | `6379` | **Compose pin** (`:-6379`) | Not operator-configurable in CD unless compose default changed | +| `REDIS_DB` | `1` | `.env` | Logical DB to avoid clashing with other apps on shared Redis | +| `CELERY_SINGLE_PROCESS` | `1` | `.env` | Weblate Celery worker process count; increase when tasks queue | +| `BOOST_ENDPOINT_THROTTLE_INFO` | `60/minute` | `.env` | Plugin rate limit (see above) | +| `BOOST_ENDPOINT_THROTTLE_ADD_OR_UPDATE` | `10/hour` | `.env` | Plugin rate limit (see above) | +| `WEBLATE_EMAIL_HOST` | `smtp.example.com` | `.env` | SMTP server; set user/password for production | +| `WEBLATE_GITHUB_USERNAME` | — | `.env` | GitHub account for VCS; required with token for add-or-update | +| `WEBLATE_GITHUB_TOKEN` | — | `.env` | GitHub PAT (`repo` scope); rotate via pre-deploy checklist | + +## Pre-Deploy Checklist + +Run before every production deploy or major upgrade. Copy into a change ticket if your process requires it. + +### Shared Redis network + +- [ ] Docker network from `REDIS_EXTERNAL_NETWORK` exists (`docker network inspect "$REDIS_EXTERNAL_NETWORK"` after sourcing `.env`) +- [ ] Redis is reachable on that network (boost-data-collector stack running, or equivalent `redis` service attached to the same network name) +- [ ] `REDIS_DB=1` in `.env` (default in `.env.example`) so Weblate does not clash with other apps on shared Redis + +### Secret rotation + +Review on a schedule or before upgrades: + +- [ ] `POSTGRES_PASSWORD` — rotate in Postgres (`ALTER USER weblate_app WITH PASSWORD '…'`) **and** in `.env`; restart stack. Updating `.env` alone is not enough. +- [ ] `WEBLATE_ADMIN_PASSWORD` — update `.env` only for initial admin provisioning; existing admins change password in the Weblate UI +- [ ] `WEBLATE_GITHUB_TOKEN` — rotate PAT in GitHub; update `.env`; restart so Celery clone/push tasks pick it up +- [ ] `WEBLATE_EMAIL_HOST_PASSWORD` — rotate SMTP credential; update `.env`; restart +- [ ] Weblate API tokens — rotate per-user tokens in the Weblate admin UI (not stored in `.env`) + +### Backup verification + +CD uses **host PostgreSQL** (`weblate_db`); there is no Postgres volume in `docker-compose.cd.yml`. + +- [ ] Confirm a recent `pg_dump` (or org backup job) of `weblate_db` exists and is restorable +- [ ] Optional spot-check: verify backup artifact timestamp/size, or `pg_dump -h localhost -U weblate_app weblate_db` succeeds +- [ ] Note: container `/app/data` (SSH keys, `known_hosts`) is not bind-mounted in CD — if Git operations fail after rollback, see [GitHub SSH host key errors](#github-ssh-host-key-errors) + +### Rollback readiness + +- [ ] Record current SHA before deploy: `git rev-parse HEAD` (or note last known-good release tag `v` from [`release.yml`](../.github/workflows/release.yml)) +- [ ] Know the rollback command (also in [Rollback (production or staging)](#rollback-production-or-staging)): + ```bash + cd /opt/cppa-weblate-plugin + git fetch origin + git checkout + docker compose -f docker/docker-compose.cd.yml --env-file .env build + docker compose -f docker/docker-compose.cd.yml --env-file .env up -d + ``` +- [ ] Plan to re-run [Post-Deploy Validation](#post-deploy-validation) after rollback +- [ ] GitHub Release tags do **not** auto-deploy; rollback is server-side git + compose only ## Build and Start @@ -341,7 +424,7 @@ Common causes: | `connection refused` on Postgres | `pg_hba.conf` or firewall blocking Docker bridge | Allow `172.17.0.0/16` in `pg_hba.conf`; reload Postgres | | `WEBLATE_ADMIN_PASSWORD … set in .env` | `.env` missing or variable unset | Ensure `.env` exists at repo root with both required secrets | | `${WEBLATE_URL_PREFIX}/healthz/` 404 | `WEBLATE_URL_PREFIX` mismatch | Ensure `.env` has `WEBLATE_URL_PREFIX` matching nginx config | -| Redis connection error | External network missing | Run `docker network create boost-data-collector_default` or start the BDC stack first | +| Redis connection error | External network missing | Start the BDC stack, or `docker network create "$REDIS_EXTERNAL_NETWORK"` (value from `.env`) | ### GitHub SSH host key errors diff --git a/scripts/README.md b/scripts/README.md index f826c6e..a6dd955 100644 --- a/scripts/README.md +++ b/scripts/README.md @@ -28,4 +28,4 @@ stack_wait_healthy 240 stack_wait_api_ready ``` -CI entrypoints use `compose up -d --wait`, `HEALTH_TIMEOUT` (smoke/auth **240**, functional **300**), `stack_create_token_retry`, and `PYTEST_PLUGIN_OPTS` for pytest timeout/reruns. See [`.github/README.md`](../.github/README.md#plugin-integration-jobs). +CI entrypoints use `compose up -d --wait`, `HEALTH_TIMEOUT` (smoke/auth **240**, functional **300**), `stack_create_token_retry`, and `PYTEST_PLUGIN_OPTS` for pytest timeout/reruns. See [`.github/WORKFLOWS.md`](../.github/WORKFLOWS.md#plugin-integration-jobs).