From ad4cea3171c98e01f38ed7b265bb9653019001d0 Mon Sep 17 00:00:00 2001 From: noctuum <25441068+noctuum@users.noreply.github.com> Date: Fri, 16 Oct 2026 02:39:12 +0500 Subject: [PATCH] fix(workflow): decide the run in a check job --- .github/workflows/check-updates.yml | 113 ++++++++++++---- README.md | 8 +- docs/WORKFLOW.md | 63 +++++---- scripts/check-all.sh | 129 ++++++++++-------- tests/publish.bats | 203 +++++++++++++++++++++------- tests/workflows.bats | 125 +++++++++++++++++ 6 files changed, 473 insertions(+), 168 deletions(-) diff --git a/.github/workflows/check-updates.yml b/.github/workflows/check-updates.yml index 9cab909..968d65b 100644 --- a/.github/workflows/check-updates.yml +++ b/.github/workflows/check-updates.yml @@ -40,23 +40,24 @@ jobs: fi echo "ref=$ref" >>"$GITHUB_OUTPUT" - # Every step that executes a PKGBUILD from the AUR runs here, without - # secrets. Only the check step, which runs no PKGBUILD, gets the - # read-only token. - prepare: + # Runs the strategies with the read-only token and decides what the run + # does. No PKGBUILD runs here, so the jobs after it trust the statuses + # this job hands over. + check: needs: image runs-on: ubuntu-latest container: ${{ needs.image.outputs.ref }} timeout-minutes: 30 outputs: - work: ${{ steps.handoff.outputs.work }} + prepare: ${{ steps.decide.outputs.prepare }} + work: ${{ steps.decide.outputs.work }} steps: - name: Install dependencies run: | pacman-key --init pacman -Sy --noconfirm --needed archlinux-keyring - pacman -Su --noconfirm --needed sudo git openssh namcap jq go-yq pacman-contrib + pacman -Su --noconfirm --needed sudo git go-yq useradd -m builder - name: Checkout @@ -72,22 +73,71 @@ jobs: chown -R builder:builder . sudo --preserve-env=GITHUB_TOKEN -u builder ./scripts/check-all.sh + - name: Decide + id: decide + run: | + # prepare runs for an outdated package, publish also for a change + # in the status table + prepare=false + if grep -qsE '^(new_version|available_downgrade): ' .status/*; then + prepare=true + fi + sudo -u builder ./scripts/update-readme.sh + work=$prepare + git diff --quiet README.md || work=true + echo "prepare=$prepare" >>"$GITHUB_OUTPUT" + echo "work=$work" >>"$GITHUB_OUTPUT" + + - name: Upload statuses + if: steps.decide.outputs.work == 'true' + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: checked + path: .status/ + include-hidden-files: true + if-no-files-found: error + retention-days: 1 + + # Every step that executes a PKGBUILD from the AUR runs here, without + # secrets and without a token. Code from one PKGBUILD can write the files + # of every package here, so this job only hands over prepared files and + # update failures, and decides nothing. + prepare: + needs: [image, check] + if: needs.check.outputs.prepare == 'true' + runs-on: ubuntu-latest + container: ${{ needs.image.outputs.ref }} + timeout-minutes: 30 + + steps: + - name: Install dependencies + run: | + pacman-key --init + pacman -Sy --noconfirm --needed archlinux-keyring + pacman -Su --noconfirm --needed sudo git openssh namcap jq go-yq pacman-contrib + useradd -m builder + + - name: Checkout + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - name: Download statuses + uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 + with: + name: checked + path: .status + - name: Prepare updates - run: sudo -u builder ./scripts/check-all.sh --prepare handoff + shell: bash + run: | + chown -R builder:builder . + # Root prefixes every line, so no output of a PKGBUILD reaches + # the runner as a workflow command + sudo -u builder ./scripts/check-all.sh --prepare handoff 2>&1 | sed 's/^/| /' - name: Hand over - id: handoff run: | - # The workspace now belongs to code from PKGBUILDs, .git/config - # included, so only builder touches it. publish runs only for an - # update or a change in the status table. - work=$(sudo -u builder bash -c ' - ./scripts/update-readme.sh >&2 - if [[ -n "$(ls -A handoff/prepared 2>/dev/null)" ]] || ! git diff --quiet README.md; then - echo true - fi - ') - # A process started by a PKGBUILD can outlive its step. Stop all # of them before root reads anything builder wrote. pkill -KILL -u builder || true @@ -110,12 +160,7 @@ jobs: done fi - if [[ "$work" == "true" ]]; then - echo "work=true" >>"$GITHUB_OUTPUT" - fi - - name: Upload handoff - if: steps.handoff.outputs.work == 'true' uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 with: name: handoff @@ -131,7 +176,6 @@ jobs: # stay out of the job with the keys. verify: needs: [image, prepare] - if: needs.prepare.outputs.work == 'true' runs-on: ubuntu-latest container: ${{ needs.image.outputs.ref }} timeout-minutes: 30 @@ -166,11 +210,17 @@ jobs: if-no-files-found: error retention-days: 1 - # Pushes what verify passed on. Nothing here executes a PKGBUILD: - # publish-pkg.sh checks the prepared files against the AUR copy. + # Pushes what verify passed on and records the statuses of the check job. + # Runs also when prepare is skipped, for a change in the status table. + # Nothing here executes a PKGBUILD: publish-pkg.sh checks the prepared + # files against the AUR copy. publish: - needs: [image, prepare, verify] - if: needs.prepare.outputs.work == 'true' + needs: [image, check, prepare, verify] + if: >- + !cancelled() && + needs.check.outputs.work == 'true' && + contains(fromJSON('["success", "skipped"]'), needs.prepare.result) && + contains(fromJSON('["success", "skipped"]'), needs.verify.result) runs-on: ubuntu-latest container: ${{ needs.image.outputs.ref }} timeout-minutes: 30 @@ -191,7 +241,14 @@ jobs: with: persist-credentials: false + - name: Download statuses + uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 + with: + name: checked + path: .status + - name: Download verified handoff + if: needs.verify.result == 'success' uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 with: name: verified diff --git a/README.md b/README.md index f4d563e..9dbec3d 100644 --- a/README.md +++ b/README.md @@ -23,14 +23,16 @@ Automated AUR package updates from a single GitHub repo. ```mermaid graph LR - A[GitHub Actions
on schedule] -->|for each package| B[prepare job, no secrets
run strategy
clone AUR repo
sed pkgver
updpkgsums
makepkg --printsrcinfo] - B -->|update or status change| V[verify job, no secrets
download the sources
check the checksums] + A[GitHub Actions
on schedule] -->|for each package| K[check job, read-only token
run strategy
decide the run] + K -->|new version| B[prepare job, no secrets
clone AUR repo
sed pkgver
updpkgsums
makepkg --printsrcinfo] + B --> V[verify job, no secrets
download the sources
check the checksums] V --> C[publish job
check the prepared files
git push to AUR] + K -->|status change only| C C -->|check or update failed| D[create GitHub Issue
with error details] C -->|done| E[update-readme.sh
commit badges + status table] ``` -Each package has a YAML config pointing to an upstream source. When a new version is detected, aurtomator clones the existing AUR repo, bumps `pkgver`, regenerates checksums and `.SRCINFO`, and pushes. A PKGBUILD is code, so these steps run in a job without secrets. A second job without secrets downloads the sources and checks the checksums, and the job that pushes checks the result once more. It does not generate PKGBUILDs or modify anything else. If anything fails, a GitHub Issue is opened automatically. +Each package has a YAML config pointing to an upstream source. A first job runs the strategies and decides what the run does. When a new version is detected, aurtomator clones the existing AUR repo, bumps `pkgver`, regenerates checksums and `.SRCINFO`, and pushes. A PKGBUILD is code, so these steps run in a job without secrets, and the jobs after it take the statuses from the first job. A second job without secrets downloads the sources and checks the checksums, and the job that pushes checks the result once more. It does not generate PKGBUILDs or modify anything else. If anything fails, a GitHub Issue is opened automatically. ## Quick start diff --git a/docs/WORKFLOW.md b/docs/WORKFLOW.md index 86c02ab..8a42525 100644 --- a/docs/WORKFLOW.md +++ b/docs/WORKFLOW.md @@ -19,15 +19,19 @@ aurtomator bumps the version of AUR packages you already maintain. It does **not ```mermaid flowchart TD - A[Schedule / manual trigger] --> P[prepare job: no secrets] - P --> B[check-all.sh with the read-only token] + A[Schedule / manual trigger] --> K[check job: read-only token, no PKGBUILD] + K --> B[check-all.sh] B --> C{for each package} C --> D[check-update.sh] D --> E[run strategy script] E --> F{version changed?} F -->|same| G[write up_to_date] F -->|new version| G2[write new_version] - C -->|all done| B2[check-all.sh --prepare without a token] + C -->|all done| W{new version or status table changed?} + W -->|no| O[done] + W -->|new version| P[prepare job: no secrets, no token] + W -->|status table only| Q + P --> B2[check-all.sh --prepare] B2 --> H[update-pkg.sh --prepare for each new_version] H --> H1[clone AUR repo over HTTPS] H1 --> H2[audit PKGBUILD] @@ -37,15 +41,13 @@ flowchart TD H5 --> H6[hand over PKGBUILD, .SRCINFO, AUR commit] H -->|failure| J[write error to .status/] D -->|failure| J - B2 -->|all done| W{update prepared or status table changed?} - W -->|no| O[done] - W -->|yes| V[verify job: no secrets] + B2 -->|all done| V[verify job: no secrets] V --> V1[check-all.sh --verify] V1 --> V2[verify-pkg.sh: download the sources, check the checksums] V2 -->|failure| V3[take the update out, write update_failed] V2 --> Q[publish job: SSH key, GPG key, write token] V3 --> Q - Q --> R[check-all.sh --publish] + Q --> R[check-all.sh --publish: statuses of the check job] R --> R1[run strategy again] R1 --> R2[publish-pkg.sh: check the prepared files against the AUR copy] R2 --> R3[git commit + push to AUR] @@ -61,7 +63,7 @@ flowchart TD - **One YAML per package.** Adding or removing a package is one file. - **Strategy-based version detection.** Each type of upstream source has its own script. A new source is a new script plus its entry in `scripts/validate-pkg.sh`. - **Patch, do not generate.** aurtomator trusts your PKGBUILD and changes only `pkgver`, `pkgrel`, checksums and `.SRCINFO`. Build steps, dependencies and install files stay untouched. -- **Three jobs, one with secrets.** A PKGBUILD is code that every step reading it executes. `prepare` checks all packages and runs those steps without secrets, and only its check step, which runs no PKGBUILD, gets the read-only workflow token. `verify` downloads the sources of each prepared update and checks its checksums, without secrets, so code from one PKGBUILD cannot forge the checksums of another package. `publish` holds the SSH key, the GPG key and the write token, never executes a PKGBUILD and never downloads a source, and checks what `verify` passes on before it pushes. See [4.4](#44-prepare-and-publish). +- **Four jobs, one with secrets.** A PKGBUILD is code that every step reading it executes. `check` runs the strategies with the read-only workflow token, executes no PKGBUILD and decides what the run does. The jobs after it take the status of every package from `check`. `prepare` runs the PKGBUILD steps without secrets and without a token, and hands over only prepared files and update failures, because code from one PKGBUILD can write the files of every package there. `verify` downloads the sources of each prepared update and checks its checksums, without secrets, so code from one PKGBUILD cannot forge the checksums of another package. `publish` holds the SSH key, the GPG key and the write token, never executes a PKGBUILD and never downloads a source, and checks what `verify` passes on before it pushes. See [4.4](#44-prepare-and-publish). - **Failure isolation.** A broken package does not block the others. Each failure is logged for its package and opens a GitHub Issue. ## 2. Package YAML Format @@ -609,7 +611,7 @@ Both count toward `pkg_ok`, so the packages badge stays `brightgreen`. Only `che ### 4.4 Prepare and publish -A PKGBUILD is a bash script, and every step that reads it executes it: the audit, `updpkgsums`, `makepkg --verifysource` and `makepkg --printsrcinfo`. Whoever can push to the AUR repo of a tracked package, a co-maintainer for example, runs code in those steps. Prepare and publish split the update so that this code never runs where the SSH key, the GPG key or a write token is. Code that runs in prepare can write the prepared files of every package, so verify checks their checksums against the sources in between. +A PKGBUILD is a bash script, and every step that reads it executes it: the audit, `updpkgsums`, `makepkg --verifysource` and `makepkg --printsrcinfo`. Whoever can push to the AUR repo of a tracked package, a co-maintainer for example, runs code in those steps. Prepare and publish split the update so that this code never runs where the SSH key, the GPG key or a write token is. Code that runs in prepare can write the prepared files and the statuses of every package, so verify checks the checksums against the sources in between, and publish takes the statuses from the check run, which executes no PKGBUILD. **Prepare**, `update-pkg.sh --prepare `, needs no secrets: @@ -637,9 +639,9 @@ A PKGBUILD is a bash script, and every step that reads it executes it: the audit `check-all.sh --prepare ` runs no strategy and needs no `GITHUB_TOKEN`. It takes the versions from the statuses that a check run, `check-all.sh` without a mode, wrote to `.status/` before it, and reads them before the first PKGBUILD runs. It prepares every package with `new_version` or `available_downgrade` into `/prepared/` and copies the statuses to `/status/`. A failed preparation becomes `update_failed`, and a package without a status from the check run becomes `check_failed`. The other statuses stay as the check run wrote them. -`check-all.sh --verify ` runs `verify-pkg.sh` for every package in `packages/` that has a directory in `/prepared/`, and runs no strategy. An update that fails is removed from `/prepared/`, its status in `/status/` becomes `update_failed`, and `/status/_error_count` grows by one. An update with exit code 3 is removed and keeps its status, and the next run prepares it again. +`check-all.sh --verify ` runs `verify-pkg.sh` for every package in `packages/` that has a directory in `/prepared/`, and runs no strategy. An update that fails is removed from `/prepared/`, its status in `/status/` becomes `update_failed`, and `/status/_error_count` grows by one. Exit code 3 counts as a failure as well: the prepare run writes `base` for every package, so a moved AUR repo looks the same as a wrong `base`. The next run prepares the package again. -`check-all.sh --publish ` builds `.status/` anew from `/status/`. The prepare run executes PKGBUILDs, so only regular files named after a package in `packages/` are taken, in the forms that run writes, and any other content becomes `check_failed`. When the script stops before that, `.status/` stays empty and the steps after it read nothing from the prepare run. Then it takes every package in `packages/` that has a directory in `/prepared/`, runs its strategy once more and publishes with the version the strategy reports now. Exit code 3 leaves the status at `new_version` or `available_downgrade` without an error, and the next run prepares the package again. +`check-all.sh --publish ` takes the statuses of the check run from `.status/`, as `--prepare` does, and writes `.status/` anew. `up_to_date` and `check_failed` stay as they are. A missing status, and one the check run does not write, become `check_failed`. For a package with `new_version` or `available_downgrade` and a directory in `/prepared/`, it runs the strategy once more and publishes with the version the strategy reports now. Exit code 3 leaves the status at `new_version` or `available_downgrade` without an error, and the next run prepares the package again. A package with `new_version` or `available_downgrade` and no prepared directory gets the `update_failed` status that the prepare or the verify run wrote for it to `/status/`, or `update_failed: prepare handed over no update`. Nothing else is taken from `/status/`, and only a regular file without a NUL byte: the prepare run executes PKGBUILDs, and code from one of them can write the files of every package. `_error_count` is the number of `check_failed` and `update_failed` statuses. ## 5. CI Workflow: check-updates.yml @@ -649,16 +651,17 @@ The workflow runs every hour (`cron: "0 * * * *"`) and by hand through `workflow ### 5.2 Permissions and container -The workflow has three jobs after a short `image` job. `image` reads the image from `.github/ci-image/Dockerfile`: `archlinux:base-devel`, pinned by digest, so a new image on Docker Hub reaches the jobs with the secrets only through a reviewed commit. Dependabot updates the digest once a month with a pull request, and `ci.yml` runs the tests in the new image before the merge, see [6.3](#63-ci-image-updates). The three jobs run in that image on `ubuntu-latest` with a 30-minute timeout, and `vercmp` from pacman, which `check-all.sh` requires, is the reason `verify` and `publish` need Arch too. Every install step upgrades `archlinux-keyring` before the system, because a pinned image ages and an old keyring rejects packages signed with a newer packager key. `makepkg` refuses to run as root, so the steps that touch packages in `prepare` and `publish` run as a `builder` user. `builder` has no `sudo` rights: the steps run as root and switch to `builder` with `sudo -u builder`. `verify` executes no PKGBUILD and runs as root. +The workflow has four jobs after a short `image` job. `image` reads the image from `.github/ci-image/Dockerfile`: `archlinux:base-devel`, pinned by digest, so a new image on Docker Hub reaches the jobs with the secrets only through a reviewed commit. Dependabot updates the digest once a month with a pull request, and `ci.yml` runs the tests in the new image before the merge, see [6.3](#63-ci-image-updates). The four jobs run in that image on `ubuntu-latest` with a 30-minute timeout, and `vercmp` from pacman, which `check-all.sh` requires, is the reason `check`, `verify` and `publish` need Arch too. Every install step upgrades `archlinux-keyring` before the system, because a pinned image ages and an old keyring rejects packages signed with a newer packager key. `makepkg` refuses to run as root, so the steps that touch packages in `prepare` and `publish` run as a `builder` user, and `check` runs the strategies as `builder` as well. `builder` has no `sudo` rights: the steps run as root and switch to `builder` with `sudo -u builder`. `verify` executes no PKGBUILD and runs as root. | Job | Permissions | Secrets | |-----|-------------|---------| | `image` | `contents: read` | none | +| `check` | `contents: read` | none | | `prepare` | `contents: read` | none | | `verify` | `contents: read` | none | | `publish` | `contents: write` for the status commit, `issues: write` for failure issues | `AUR_SSH_KEY`, `GPG_SIGNING_KEY`, `AUR_GIT_NAME`, `AUR_GIT_EMAIL` | -`verify` and `publish` run only when `prepare` hands over a prepared update or a change in the status table of `README.md`, and `publish` only after `verify` succeeds. In most hours nothing changes, and both are skipped. +`check` decides the run. `prepare` and `verify` run only when a package has a new version. `publish` runs for a new version and also for a change in the status table of `README.md`, but not after `prepare` or `verify` failed. In most hours nothing changes, and all three are skipped. ### 5.3 Steps @@ -669,12 +672,19 @@ Every job starts with **Checkout:** `actions/checkout`, pinned by commit SHA lik 1. **Checkout:** `.github/ci-image` only. 2. **Read image:** takes the `FROM` line of `.github/ci-image/Dockerfile` and fails unless it is `archlinux:base-devel@sha256:` with a 64-character digest. +`check`: + +1. **Install dependencies:** `sudo git go-yq`, and the `builder` user. +2. **Check updates:** gives the workspace to `builder` and runs `check-all.sh` as `builder`, with `GITHUB_TOKEN` set to the read-only workflow token. `sudo --preserve-env` passes the token, so it does not appear on a command line. +3. **Decide:** sets the output `prepare` when a status is `new_version` or `available_downgrade`. It runs `update-readme.sh` as `builder` and sets the output `work` for `prepare` and for a change in the status table. +4. **Upload statuses:** with `work`, uploads `.status/` as the `checked` artifact with a retention of 1 day. + `prepare`: 1. **Install dependencies:** `sudo git openssh namcap jq go-yq pacman-contrib`, and the `builder` user. -2. **Check updates:** gives the workspace to `builder` and runs `check-all.sh` as `builder`, with `GITHUB_TOKEN` set to the read-only workflow token. `sudo --preserve-env` passes the token, so it does not appear on a command line. No PKGBUILD runs in this step. -3. **Prepare updates:** runs `check-all.sh --prepare handoff` as `builder`, without a token. Code from a PKGBUILD can read the environment of every running process of `builder` through `/proc//environ`, so the token exists only in the step before. -4. **Hand over:** as `builder`, runs `update-readme.sh` to find out whether the status table changes. Code from a PKGBUILD has run in the workspace by now, `.git/config` included, so root does not run git there. A process started by a PKGBUILD can outlive its step, so the step then kills every process of `builder`. Only after that does root copy the regular files at `prepared//PKGBUILD`, `prepared//.SRCINFO`, `prepared//base` and `status/` from `handoff/` to `upload/`. `find` does not follow symlinks and separates paths with NUL, because a name may hold a newline, so a link to a file outside the workspace does not reach the artifact. A package name never starts with a dot, so `.` and `..` match no path. +2. **Download statuses:** the `checked` artifact to `.status/`. +3. **Prepare updates:** gives the workspace to `builder` and runs `check-all.sh --prepare handoff` as `builder`, without a token. Code from a PKGBUILD can read the environment of every running process of `builder` through `/proc//environ`, so the token stays in `check`. Root prefixes every line of the output with `|` and a space, so nothing a PKGBUILD prints reaches the runner as a workflow command such as `::add-mask::`. The step runs in bash with `pipefail`, so a failure of `check-all.sh` fails it. +4. **Hand over:** code from a PKGBUILD has run in the workspace by now, so the step runs nothing from it. A process started by a PKGBUILD can outlive its step, so the step kills every process of `builder`. Only after that does root copy the regular files at `prepared//PKGBUILD`, `prepared//.SRCINFO`, `prepared//base` and `status/` from `handoff/` to `upload/`. `find` does not follow symlinks and separates paths with NUL, because a name may hold a newline, so a link to a file outside the workspace does not reach the artifact. A package name never starts with a dot, so `.` and `..` match no path. 5. **Upload handoff:** uploads `upload/` as the `handoff` artifact with `include-hidden-files: true`, because `.SRCINFO` starts with a dot, and a retention of 1 day. `verify`: @@ -687,16 +697,17 @@ Every job starts with **Checkout:** `actions/checkout`, pinned by commit SHA lik `publish`: 1. **Install dependencies:** `sudo git github-cli openssh gnupg jq go-yq`, and the `builder` user. -2. **Download verified handoff:** the `verified` artifact to `handoff/`. -3. **Setup workspace:** git identity for `builder` from the `AUR_GIT_NAME` and `AUR_GIT_EMAIL` secrets, default `aurtomator`. -4. **Setup AUR SSH:** writes `AUR_SSH_KEY` to `/home/builder/.ssh/aur`, writes the Ed25519 host key that publishes to `known_hosts`, and writes an SSH config with `StrictHostKeyChecking yes` and connection multiplexing. A host that answers with another key fails with `Host key verification failed`. When the AUR changes its host key, update the key in `check-updates.yml`. -5. **Setup GPG:** with `GPG_SIGNING_KEY` set, imports the key for `builder`, passes `GPG_KEY_ID` to the later steps and enables commit signing. Without it, the step is skipped. -6. **Publish updates:** runs `check-all.sh --publish handoff` as `builder`, with `GPG_KEY_ID`, `GIT_AUTHOR_NAME`, `GIT_AUTHOR_EMAIL` and `GITHUB_TOKEN` set to the workflow token. It builds `.status/` from the checked statuses and checks the prepared files before it pushes, see [4.4](#44-prepare-and-publish). -7. **Update README status table:** runs `update-readme.sh` as `builder`. -8. **Commit changes via API:** see [5.5](#55-verified-commit). -9. **Create issues for failed packages:** see [5.6](#56-failure-issues). - -Steps 8 and 9 run as root with `GH_TOKEN` set to the workflow token, `github.token`. +2. **Download statuses:** the `checked` artifact to `.status/`. +3. **Download verified handoff:** the `verified` artifact to `handoff/`, when `verify` ran. +4. **Setup workspace:** git identity for `builder` from the `AUR_GIT_NAME` and `AUR_GIT_EMAIL` secrets, default `aurtomator`. +5. **Setup AUR SSH:** writes `AUR_SSH_KEY` to `/home/builder/.ssh/aur`, writes the Ed25519 host key that publishes to `known_hosts`, and writes an SSH config with `StrictHostKeyChecking yes` and connection multiplexing. A host that answers with another key fails with `Host key verification failed`. When the AUR changes its host key, update the key in `check-updates.yml`. +6. **Setup GPG:** with `GPG_SIGNING_KEY` set, imports the key for `builder`, passes `GPG_KEY_ID` to the later steps and enables commit signing. Without it, the step is skipped. +7. **Publish updates:** runs `check-all.sh --publish handoff` as `builder`, with `GPG_KEY_ID`, `GIT_AUTHOR_NAME`, `GIT_AUTHOR_EMAIL` and `GITHUB_TOKEN` set to the workflow token. It keeps the statuses of `check` and checks the prepared files before it pushes, see [4.4](#44-prepare-and-publish). +8. **Update README status table:** runs `update-readme.sh` as `builder`. +9. **Commit changes via API:** see [5.5](#55-verified-commit). +10. **Create issues for failed packages:** see [5.6](#56-failure-issues). + +Steps 9 and 10 run as root with `GH_TOKEN` set to the workflow token, `github.token`. ### 5.4 README update diff --git a/scripts/check-all.sh b/scripts/check-all.sh index 45c8760..4291964 100755 --- a/scripts/check-all.sh +++ b/scripts/check-all.sh @@ -11,8 +11,8 @@ # strategies, and copies the statuses to /status # With --verify : runs verify-pkg.sh for each package in /prepared # and takes out the ones that fail, without running the strategies -# With --publish : runs publish-pkg.sh for each package in /prepared -# and takes the other statuses from /status +# With --publish : keeps the statuses of the check run before it, and +# runs publish-pkg.sh for each package that run found outdated # With --dry-run: runs update-pkg.sh or publish-pkg.sh in dry-run mode # # Writes per-package status to .status/ directory for README generation @@ -58,10 +58,12 @@ while [[ $# -gt 0 ]]; do esac done -# --verify checks every update in /prepared with verify-pkg.sh. It takes -# out an update that fails, with an update_failed status, and one the AUR -# moved under, with its status kept, so --publish skips both. Only package -# names from packages/ reach a path. +# --verify checks every update in /prepared with verify-pkg.sh and takes +# out an update that fails, with an update_failed status, so --publish skips +# it. An AUR repo that moved counts as a failure: the prepare run writes the +# base of every package, so a move looks the same as a wrong base. The next +# run prepares the package again. Only package names from packages/ reach a +# path. if [[ "$MODE" == "verify" ]]; then failed=0 for pkg_file in "${AURTOMATOR_DIR}"/packages/*.yml; do @@ -81,9 +83,6 @@ if [[ "$MODE" == "verify" ]]; then case "$verify_rc" in 0) ;; - 3) - rm -rf "$prepared" - ;; *) log_err "Failed to verify $pkg_name" rm -rf "$prepared" @@ -111,13 +110,13 @@ fi # Status directory for per-package results STATUS_DIR="${AURTOMATOR_DIR}/.status" -# --prepare runs code from PKGBUILDs, so the workflow gives it no -# GITHUB_TOKEN and it runs no strategy. It takes the versions from the -# statuses of the check run before it, read here before any PKGBUILD runs. +# --prepare and --publish take the statuses of the check run before them, +# read here before anything else runs. --prepare runs code from PKGBUILDs, +# so the workflow gives it no GITHUB_TOKEN and it runs no strategy. declare -A CHECKED=() -if [[ "$MODE" == "prepare" ]]; then +if [[ "$MODE" == "prepare" || "$MODE" == "publish" ]] && [[ ! -L "$STATUS_DIR" ]]; then for status_file in "$STATUS_DIR"/*; do - [[ -f "$status_file" ]] || continue + [[ -f "$status_file" && ! -L "$status_file" ]] || continue CHECKED["${status_file##*/}"]=$(<"$status_file") done fi @@ -125,38 +124,24 @@ fi rm -rf "$STATUS_DIR" mkdir -p "$STATUS_DIR" -# --publish takes the statuses from the prepare run, which executed -# PKGBUILDs while it wrote them. Only statuses of known packages, in the -# forms that run writes, are copied. When this script stops early, .status/ -# stays empty, so the steps after it read nothing from that run. -if [[ "$MODE" == "publish" && -d "${WORK_DIR}/status" && ! -L "${WORK_DIR}/status" ]]; then - known_names=$(for f in "${AURTOMATOR_DIR}"/packages/*.yml; do - [[ -f "$f" ]] && pkg_get "$f" .name - done) - for status_file in "${WORK_DIR}/status"/*; do - [[ -f "$status_file" && ! -L "$status_file" ]] || continue - name="${status_file##*/}" - if [[ ! "$name" =~ ^[A-Za-z0-9@._+-]+$ ]] || ! grep -qxF -- "$name" <<<"$known_names"; then - continue +# The update failure that the prepare or the verify run recorded for a +# package the check run found outdated. Both runs share one user with code +# from every PKGBUILD, so nothing else of theirs is taken, and a package +# they handed over nothing for fails as well. +handed_failure() { + local status_dir="${WORK_DIR}/status" + local file="${status_dir}/$1" + if [[ -d "$status_dir" && ! -L "$status_dir" && -f "$file" && ! -L "$file" ]] && + ! _has_nul "$file"; then + local handed + handed=$(<"$file") + if [[ "$handed" == update_failed:* ]]; then + printf '%s\n' "$handed" + return fi - status="check_failed: invalid status from the prepare run" - if ! _has_nul "$status_file"; then - # The whole file, as update-readme.sh and the issue step read it - handed_over=$(<"$status_file") - case "$handed_over" in - up_to_date | check_failed:* | update_failed:*) - status="$handed_over" - ;; - "new_version: "* | "available_downgrade: "*) - if validate_pkgver "${handed_over#*: }" 2>/dev/null; then - status="$handed_over" - fi - ;; - esac - fi - printf '%s\n' "$status" >"${STATUS_DIR}/${name}" - done -fi + fi + echo "update_failed: prepare handed over no update" +} updated=0 checked=0 @@ -186,10 +171,35 @@ for pkg_file in $(printf '%s\n' "${AURTOMATOR_DIR}"/packages/*.yml | sort); do [[ -f "$pkg_file" ]] || continue pkg_name=$(pkg_get "$pkg_file" .name) - # Only package names from packages/ reach a path, never a name from WORK_DIR - if [[ "$MODE" == "publish" ]] && - [[ -L "${WORK_DIR}/prepared" || ! -d "${WORK_DIR}/prepared/${pkg_name}" ]]; then - continue + # --publish keeps the status of the check run and publishes only a package + # that run found outdated. Only package names from packages/ reach a path, + # never a name from WORK_DIR. + if [[ "$MODE" == "publish" ]]; then + found="${CHECKED[$pkg_name]:-}" + case "$found" in + "new_version: "* | "available_downgrade: "*) + if ! validate_pkgver "${found#*: }" 2>/dev/null; then + echo "check_failed: invalid status from the check run" >"${STATUS_DIR}/${pkg_name}" + continue + fi + if [[ -L "${WORK_DIR}/prepared" || ! -d "${WORK_DIR}/prepared/${pkg_name}" ]]; then + handed_failure "$pkg_name" >"${STATUS_DIR}/${pkg_name}" + continue + fi + ;; + up_to_date | check_failed:*) + printf '%s\n' "$found" >"${STATUS_DIR}/${pkg_name}" + continue + ;; + "") + echo "check_failed: not checked before --publish" >"${STATUS_DIR}/${pkg_name}" + continue + ;; + *) + echo "check_failed: invalid status from the check run" >"${STATUS_DIR}/${pkg_name}" + continue + ;; + esac fi ((checked++)) || true @@ -286,19 +296,20 @@ for pkg_file in $(printf '%s\n' "${AURTOMATOR_DIR}"/packages/*.yml | sort); do esac done +# --publish counts the failures among the statuses it wrote +if [[ "$MODE" == "publish" ]]; then + errors=0 + for status_file in "$STATUS_DIR"/*; do + [[ -f "$status_file" ]] || continue + case "$(<"$status_file")" in + check_failed:* | update_failed:*) ((errors++)) || true ;; + esac + done +fi + echo "" log_info "Checked: $checked, Updated: $updated, Errors: $errors" -# --publish adds its errors to those of the prepare run. The file comes from -# that run, so it is read only when it holds a plain number. -handed_count="${WORK_DIR}/status/_error_count" -if [[ "$MODE" == "publish" && -f "$handed_count" && ! -L "$handed_count" && ! -L "${WORK_DIR}/status" ]]; then - prepare_errors=$(<"$handed_count") - if [[ "$prepare_errors" =~ ^[0-9]+$ ]]; then - errors=$((errors + prepare_errors)) - fi -fi - # Write summary for downstream steps echo "${errors}" >"${STATUS_DIR}/_error_count" diff --git a/tests/publish.bats b/tests/publish.bats index 626ea60..8c83c39 100644 --- a/tests/publish.bats +++ b/tests/publish.bats @@ -677,16 +677,22 @@ current: '1.0.0'" [ "$(cat "$TEST_TMPDIR/.status/pkg")" = "update_failed: audit failed" ] } -@test "check-all --publish: publishes with the version upstream reports now and imports other statuses" { +# Helper: write the status that the check job hands over for a package +# Usage: checked_status +checked_status() { + printf '%s\n' "$2" >"$TEST_TMPDIR/.status/$1" +} + +@test "check-all --publish: publishes with the version upstream reports now and keeps the checked statuses" { create_test_package "pkg" "github-release" "upstream: project: owner/repo current: '1.0.0'" create_test_package "other" "github-release" "upstream: project: owner/repo current: '2.0.0'" - mkdir -p "$TEST_TMPDIR/out/prepared/pkg" "$TEST_TMPDIR/out/status" - echo "new_version: 2.0.0" >"$TEST_TMPDIR/out/status/pkg" - echo "up_to_date" >"$TEST_TMPDIR/out/status/other" + checked_status pkg "new_version: 2.0.0" + checked_status other "up_to_date" + mkdir -p "$TEST_TMPDIR/out/prepared/pkg" stub_script publish-pkg.sh 0 run "$TEST_TMPDIR/scripts/check-all.sh" --publish "$TEST_TMPDIR/out" [ "$status" -eq 0 ] @@ -695,19 +701,111 @@ current: '2.0.0'" [ "$(cat "$TEST_TMPDIR/.status/other")" = "up_to_date" ] } -@test "check-all --publish: starts from an empty status directory" { +@test "check-all --publish: keeps a failed check" { create_test_package "pkg" "github-release" "upstream: project: owner/repo current: '2.0.0'" - echo "update_failed: stale" >"$TEST_TMPDIR/.status/pkg" + checked_status pkg "check_failed: API down" + run "$TEST_TMPDIR/scripts/check-all.sh" --publish "$TEST_TMPDIR/out" + [ "$status" -eq 0 ] + [ "$(cat "$TEST_TMPDIR/.status/pkg")" = "check_failed: API down" ] + [ "$(cat "$TEST_TMPDIR/.status/_error_count")" = "1" ] +} + +@test "check-all --publish: marks a package the check run left out" { + create_test_package "pkg" "github-release" "upstream: + project: owner/repo +current: '2.0.0'" + run "$TEST_TMPDIR/scripts/check-all.sh" --publish "$TEST_TMPDIR/out" + [ "$(cat "$TEST_TMPDIR/.status/pkg")" = "check_failed: not checked before --publish" ] +} + +@test "check-all --publish: replaces a checked status the check run does not write" { + create_test_package "pkg" "github-release" "upstream: + project: owner/repo +current: '2.0.0'" + checked_status pkg "update_failed: stale" + run "$TEST_TMPDIR/scripts/check-all.sh" --publish "$TEST_TMPDIR/out" + [ "$(cat "$TEST_TMPDIR/.status/pkg")" = "check_failed: invalid status from the check run" ] +} + +@test "check-all --publish: replaces a checked status with an unsafe version" { + create_test_package "pkg" "github-release" "upstream: + project: owner/repo +current: '1.0.0'" + checked_status pkg "new_version: 1.0](https://evil.example)" + mkdir -p "$TEST_TMPDIR/out/prepared/pkg" + stub_script publish-pkg.sh 0 + run "$TEST_TMPDIR/scripts/check-all.sh" --publish "$TEST_TMPDIR/out" + [ ! -e "$TEST_TMPDIR/publish-pkg.sh.args" ] + [ "$(cat "$TEST_TMPDIR/.status/pkg")" = "check_failed: invalid status from the check run" ] +} + +@test "check-all --publish: replaces a checked status with extra lines" { + create_test_package "pkg" "github-release" "upstream: + project: owner/repo +current: '1.0.0'" + checked_status pkg $'new_version: 2.0.0\n| [x](https://evil.example) |' + run "$TEST_TMPDIR/scripts/check-all.sh" --publish "$TEST_TMPDIR/out" + [ "$(cat "$TEST_TMPDIR/.status/pkg")" = "check_failed: invalid status from the check run" ] +} + +@test "check-all --publish: publishes only packages the check run found outdated" { + create_test_package "pkg" "github-release" "upstream: + project: owner/repo +current: '1.0.0'" + checked_status pkg "up_to_date" + mkdir -p "$TEST_TMPDIR/out/prepared/pkg" + stub_script publish-pkg.sh 0 + run "$TEST_TMPDIR/scripts/check-all.sh" --publish "$TEST_TMPDIR/out" + [ ! -e "$TEST_TMPDIR/publish-pkg.sh.args" ] + [ "$(cat "$TEST_TMPDIR/.status/pkg")" = "up_to_date" ] +} + +@test "check-all --publish: records an outdated package without an update as failed" { + create_test_package "pkg" "github-release" "upstream: + project: owner/repo +current: '1.0.0'" + checked_status pkg "new_version: 2.0.0" + run "$TEST_TMPDIR/scripts/check-all.sh" --publish "$TEST_TMPDIR/out" + [ "$status" -eq 0 ] + [ "$(cat "$TEST_TMPDIR/.status/pkg")" = "update_failed: prepare handed over no update" ] + [ "$(cat "$TEST_TMPDIR/.status/_error_count")" = "1" ] +} + +@test "check-all --publish: keeps the update failure the prepare run recorded" { + create_test_package "pkg" "github-release" "upstream: + project: owner/repo +current: '1.0.0'" + checked_status pkg "new_version: 2.0.0" + mkdir -p "$TEST_TMPDIR/out/status" + printf 'update_failed: audit failed\nline two\n' >"$TEST_TMPDIR/out/status/pkg" run "$TEST_TMPDIR/scripts/check-all.sh" --publish "$TEST_TMPDIR/out" - [ ! -e "$TEST_TMPDIR/.status/pkg" ] + [ "$(cat "$TEST_TMPDIR/.status/pkg")" = $'update_failed: audit failed\nline two' ] +} + +@test "check-all --publish: takes no other status from the prepare run" { + create_test_package "pkg" "github-release" "upstream: + project: owner/repo +current: '1.0.0'" + create_test_package "other" "github-release" "upstream: + project: owner/repo +current: '2.0.0'" + checked_status pkg "new_version: 2.0.0" + checked_status other "up_to_date" + mkdir -p "$TEST_TMPDIR/out/status" + echo "up_to_date" >"$TEST_TMPDIR/out/status/pkg" + echo "check_failed: forged" >"$TEST_TMPDIR/out/status/other" + run "$TEST_TMPDIR/scripts/check-all.sh" --publish "$TEST_TMPDIR/out" + [ "$(cat "$TEST_TMPDIR/.status/pkg")" = "update_failed: prepare handed over no update" ] + [ "$(cat "$TEST_TMPDIR/.status/other")" = "up_to_date" ] } @test "check-all --publish: leaves a deferred package as available, without an error" { create_test_package "pkg" "github-release" "upstream: project: owner/repo current: '1.0.0'" + checked_status pkg "new_version: 2.0.0" mkdir -p "$TEST_TMPDIR/out/prepared/pkg" stub_script publish-pkg.sh 3 run "$TEST_TMPDIR/scripts/check-all.sh" --publish "$TEST_TMPDIR/out" @@ -719,6 +817,7 @@ current: '1.0.0'" create_test_package "pkg" "github-release" "upstream: project: owner/repo current: '1.0.0'" + checked_status pkg "new_version: 2.0.0" mkdir -p "$TEST_TMPDIR/out/prepared/pkg" stub_script publish-pkg.sh 1 "changes lines outside pkgver" run "$TEST_TMPDIR/scripts/check-all.sh" --publish "$TEST_TMPDIR/out" @@ -730,6 +829,7 @@ current: '1.0.0'" create_test_package "pkg" "github-release" "upstream: project: owner/repo current: '2.0.0'" + checked_status pkg "up_to_date" mkdir -p "$TEST_TMPDIR/out/prepared/evil" stub_script publish-pkg.sh 0 run "$TEST_TMPDIR/scripts/check-all.sh" --publish "$TEST_TMPDIR/out" @@ -741,83 +841,75 @@ current: '2.0.0'" create_test_package "pkg" "github-release" "upstream: project: owner/repo current: '1.0.0'" + checked_status pkg "new_version: 2.0.0" mkdir -p "$TEST_TMPDIR/elsewhere/pkg" "$TEST_TMPDIR/out" ln -s "$TEST_TMPDIR/elsewhere" "$TEST_TMPDIR/out/prepared" stub_script publish-pkg.sh 0 run "$TEST_TMPDIR/scripts/check-all.sh" --publish "$TEST_TMPDIR/out" [ ! -f "$TEST_TMPDIR/publish-pkg.sh.args" ] + [ "$(cat "$TEST_TMPDIR/.status/pkg")" = "update_failed: prepare handed over no update" ] } @test "check-all --publish: does not import statuses of names that are not packages" { create_test_package "pkg" "github-release" "upstream: project: owner/repo current: '2.0.0'" + checked_status pkg "up_to_date" mkdir -p "$TEST_TMPDIR/out/status" echo "update_failed: spoofed" >"$TEST_TMPDIR/out/status/evil" - echo "up_to_date" >"$TEST_TMPDIR/out/status/pkg" run "$TEST_TMPDIR/scripts/check-all.sh" --publish "$TEST_TMPDIR/out" [ ! -e "$TEST_TMPDIR/.status/evil" ] [ "$(cat "$TEST_TMPDIR/.status/pkg")" = "up_to_date" ] } -@test "check-all --publish: does not import a status directory that is a symlink" { +@test "check-all --publish: does not read a handed over status directory that is a symlink" { create_test_package "pkg" "github-release" "upstream: project: owner/repo -current: '2.0.0'" +current: '1.0.0'" + checked_status pkg "new_version: 2.0.0" mkdir -p "$TEST_TMPDIR/elsewhere" "$TEST_TMPDIR/out" - echo "up_to_date" >"$TEST_TMPDIR/elsewhere/pkg" + echo "update_failed: from elsewhere" >"$TEST_TMPDIR/elsewhere/pkg" ln -s "$TEST_TMPDIR/elsewhere" "$TEST_TMPDIR/out/status" run "$TEST_TMPDIR/scripts/check-all.sh" --publish "$TEST_TMPDIR/out" - [ ! -e "$TEST_TMPDIR/.status/pkg" ] -} - -@test "check-all --publish: replaces a status the prepare run does not write" { - create_test_package "pkg" "github-release" "upstream: - project: owner/repo -current: '2.0.0'" - mkdir -p "$TEST_TMPDIR/out/status" - echo "updated: 9.9.9" >"$TEST_TMPDIR/out/status/pkg" - run "$TEST_TMPDIR/scripts/check-all.sh" --publish "$TEST_TMPDIR/out" - [[ "$(cat "$TEST_TMPDIR/.status/pkg")" == "check_failed:"* ]] + [ "$(cat "$TEST_TMPDIR/.status/pkg")" = "update_failed: prepare handed over no update" ] } -@test "check-all --publish: replaces a status with an unsafe version" { +@test "check-all --publish: does not follow symlinks among the handed over statuses" { create_test_package "pkg" "github-release" "upstream: project: owner/repo -current: '2.0.0'" - mkdir -p "$TEST_TMPDIR/out/status" - echo "new_version: 1.0](https://evil.example)" >"$TEST_TMPDIR/out/status/pkg" - run "$TEST_TMPDIR/scripts/check-all.sh" --publish "$TEST_TMPDIR/out" - [[ "$(cat "$TEST_TMPDIR/.status/pkg")" == "check_failed:"* ]] -} - -@test "check-all --publish: replaces a status with extra lines" { - create_test_package "pkg" "github-release" "upstream: +current: '1.0.0'" + create_test_package "other" "github-release" "upstream: project: owner/repo -current: '2.0.0'" - mkdir -p "$TEST_TMPDIR/out/status" - printf 'new_version: 2.0.0\n| [x](https://evil.example) |\n' >"$TEST_TMPDIR/out/status/pkg" +current: '1.0.0'" + checked_status pkg "new_version: 2.0.0" + checked_status other "new_version: 2.0.0" + mkdir -p "$TEST_TMPDIR/out/prepared/pkg" "$TEST_TMPDIR/out/status" + echo "update_failed: from a link" >"$TEST_TMPDIR/target-status" + echo "keep" >"$TEST_TMPDIR/target-count" + ln -s "$TEST_TMPDIR/target-status" "$TEST_TMPDIR/out/status/pkg" + ln -s "$TEST_TMPDIR/target-status" "$TEST_TMPDIR/out/status/other" + ln -s "$TEST_TMPDIR/target-count" "$TEST_TMPDIR/out/status/_error_count" + stub_script publish-pkg.sh 0 run "$TEST_TMPDIR/scripts/check-all.sh" --publish "$TEST_TMPDIR/out" - [[ "$(cat "$TEST_TMPDIR/.status/pkg")" == "check_failed:"* ]] - ! grep -q evil "$TEST_TMPDIR/.status/pkg" + [ "$(cat "$TEST_TMPDIR/target-status")" = "update_failed: from a link" ] + [ "$(cat "$TEST_TMPDIR/target-count")" = "keep" ] + [ "$(cat "$TEST_TMPDIR/.status/pkg")" = "updated: 2.0.0" ] + [ "$(cat "$TEST_TMPDIR/.status/other")" = "update_failed: prepare handed over no update" ] } -@test "check-all --publish: does not follow symlinks among the handed over statuses" { +@test "check-all --publish: does not read a checked status that is a symlink" { create_test_package "pkg" "github-release" "upstream: project: owner/repo current: '1.0.0'" - mkdir -p "$TEST_TMPDIR/out/prepared/pkg" "$TEST_TMPDIR/out/status" - echo "keep" >"$TEST_TMPDIR/target-status" - echo "keep" >"$TEST_TMPDIR/target-count" - ln -s "$TEST_TMPDIR/target-status" "$TEST_TMPDIR/out/status/pkg" - ln -s "$TEST_TMPDIR/target-count" "$TEST_TMPDIR/out/status/_error_count" + echo "new_version: 2.0.0" >"$TEST_TMPDIR/target-status" ln -s "$TEST_TMPDIR/target-status" "$TEST_TMPDIR/.status/pkg" + mkdir -p "$TEST_TMPDIR/out/prepared/pkg" stub_script publish-pkg.sh 0 run "$TEST_TMPDIR/scripts/check-all.sh" --publish "$TEST_TMPDIR/out" - [ "$(cat "$TEST_TMPDIR/target-status")" = "keep" ] - [ "$(cat "$TEST_TMPDIR/target-count")" = "keep" ] + [ ! -e "$TEST_TMPDIR/publish-pkg.sh.args" ] [ ! -L "$TEST_TMPDIR/.status/pkg" ] - [ "$(cat "$TEST_TMPDIR/.status/pkg")" = "updated: 2.0.0" ] + [ "$(cat "$TEST_TMPDIR/target-status")" = "new_version: 2.0.0" ] + [ "$(cat "$TEST_TMPDIR/.status/pkg")" = "check_failed: not checked before --publish" ] } @test "check-all --publish: replaces a status directory that is a symlink" { @@ -833,16 +925,22 @@ current: '2.0.0'" [ -z "$(ls -A "$TEST_TMPDIR/elsewhere")" ] } -@test "check-all --publish: ignores an error count that is not a number" { +@test "check-all --publish: counts the errors itself" { create_test_package "pkg" "github-release" "upstream: project: owner/repo current: '1.0.0'" + create_test_package "broken" "github-release" "upstream: + project: owner/repo +current: '1.0.0'" + checked_status pkg "new_version: 2.0.0" + checked_status broken "check_failed: API down" + echo 7 >"$TEST_TMPDIR/.status/_error_count" mkdir -p "$TEST_TMPDIR/out/prepared/pkg" "$TEST_TMPDIR/out/status" echo 'a[$(touch '"$TEST_TMPDIR"'/pwned)]' >"$TEST_TMPDIR/out/status/_error_count" stub_script publish-pkg.sh 0 run "$TEST_TMPDIR/scripts/check-all.sh" --publish "$TEST_TMPDIR/out" [ ! -f "$TEST_TMPDIR/pwned" ] - [ "$(cat "$TEST_TMPDIR/.status/_error_count")" = "0" ] + [ "$(cat "$TEST_TMPDIR/.status/_error_count")" = "1" ] } # === check-all.sh --verify === @@ -880,14 +978,15 @@ current: '1.0.0'" [ "$(cat "$TEST_TMPDIR/out/status/_error_count")" = "2" ] } -@test "check-all --verify: takes out an update the AUR moved under and keeps its status" { +# The prepare run writes the base of every package, so a moved AUR repo and +# a wrong base look the same +@test "check-all --verify: takes out an update the AUR moved under, with update_failed" { setup_check_verify - stub_script verify-pkg.sh 3 + stub_script verify-pkg.sh 3 "the AUR repo moved" run "$TEST_TMPDIR/scripts/check-all.sh" --verify "$TEST_TMPDIR/out" [ "$status" -eq 0 ] [ ! -e "$TEST_TMPDIR/out/prepared/pkg" ] - [ "$(cat "$TEST_TMPDIR/out/status/pkg")" = "new_version: 2.0.0" ] - [ "$(cat "$TEST_TMPDIR/out/status/_error_count")" = "1" ] + [ "$(cat "$TEST_TMPDIR/out/status/pkg")" = "update_failed: the AUR repo moved" ] } @test "check-all --verify: verifies only prepared packages from packages/" { diff --git a/tests/workflows.bats b/tests/workflows.bats index 007327d..9bd6bfd 100644 --- a/tests/workflows.bats +++ b/tests/workflows.bats @@ -154,3 +154,128 @@ EOF run grep -c '^pr merge' "$GH_LOG" [ "$output" = "0" ] } + +# === check-updates.yml: who decides what === + +CHECK_UPDATES="$GITHUB_DIR/workflows/check-updates.yml" + +# Prints the run block of step $2 in job $1 of check-updates.yml +step_run() { + yq ".jobs.$1.steps[] | select(.name == \"$2\") | .run" "$CHECK_UPDATES" +} + +@test "check-updates: the check job decides the run and executes no PKGBUILD" { + run yq '.jobs.check.steps[] | select(has("run")) | .run' "$CHECK_UPDATES" + [ "$status" -eq 0 ] + [[ "$output" == *"./scripts/check-all.sh"* ]] + [[ "$output" != *"--prepare"* ]] + [[ "$output" != *"update-pkg.sh"* ]] + run yq '.jobs.check.outputs | keys | join(" ")' "$CHECK_UPDATES" + [ "$output" = "prepare work" ] +} + +# Runs the Decide step in a workspace with the statuses given as name=status +# pairs, a sudo that does nothing and a git whose diff reports a README +# change when $README_CHANGED is set; prints the step outputs +run_decide_step() { + local ws="$BATS_TEST_TMPDIR/ws" bin="$BATS_TEST_TMPDIR/bin" pair + mkdir -p "$ws/.status" "$bin" + for pair in "$@"; do + printf '%s\n' "${pair#*=}" >"$ws/.status/${pair%%=*}" + done + printf '#!/bin/sh\nexit 0\n' >"$bin/sudo" + printf '#!/bin/sh\n[ -z "$README_CHANGED" ]\n' >"$bin/git" + chmod +x "$bin/sudo" "$bin/git" + step_run check Decide >"$BATS_TEST_TMPDIR/step.sh" + : >"$BATS_TEST_TMPDIR/output" + (cd "$ws" && PATH="$bin:$PATH" GITHUB_OUTPUT="$BATS_TEST_TMPDIR/output" \ + sh -e "$BATS_TEST_TMPDIR/step.sh") >/dev/null + cat "$BATS_TEST_TMPDIR/output" +} + +@test "check-updates: Decide prepares when a package is outdated" { + run run_decide_step pkg="new_version: 2.0.0" other=up_to_date _error_count=0 + [ "$status" -eq 0 ] + [ "$output" = $'prepare=true\nwork=true' ] + run run_decide_step pkg="available_downgrade: 1.0.0" + [ "$output" = $'prepare=true\nwork=true' ] +} + +@test "check-updates: Decide publishes only a change of the status table" { + README_CHANGED=1 run run_decide_step pkg="check_failed: API down" + [ "$status" -eq 0 ] + [ "$output" = $'prepare=false\nwork=true' ] +} + +@test "check-updates: Decide does nothing when nothing changed" { + run run_decide_step pkg=up_to_date _error_count=0 + [ "$status" -eq 0 ] + [ "$output" = $'prepare=false\nwork=false' ] +} + +@test "check-updates: prepare runs only for outdated packages and decides nothing" { + run yq '.jobs.prepare.if' "$CHECK_UPDATES" + [ "$output" = "needs.check.outputs.prepare == 'true'" ] + run yq '.jobs.prepare.outputs' "$CHECK_UPDATES" + [ "$output" = "null" ] +} + +@test "check-updates: Hand over runs nothing from the workspace" { + run step_run prepare "Hand over" + [ "$status" -eq 0 ] + [ -n "$output" ] + [[ "$output" != *"./scripts/"* ]] + [[ "$output" != *"sudo -u builder"* ]] + [[ "$output" != *"GITHUB_OUTPUT"* ]] +} + +# Runs the Prepare updates step with a sudo that prints $SUDO_OUT and exits +# with $SUDO_RC, in the shell the step names. chown does nothing. +run_prepare_step() { + local bin="$BATS_TEST_TMPDIR/bin" + mkdir -p "$bin" + printf '#!/bin/sh\nexit 0\n' >"$bin/chown" + chmod +x "$bin/chown" + cat >"$bin/sudo" <<'EOF' +#!/usr/bin/env bash +printf '%s\n' "$SUDO_OUT" +printf '%s\n' "$SUDO_OUT" >&2 +exit "${SUDO_RC:-0}" +EOF + chmod +x "$bin/sudo" + step_run prepare "Prepare updates" >"$BATS_TEST_TMPDIR/step.sh" + run yq '.jobs.prepare.steps[] | select(.name == "Prepare updates") | .shell' "$CHECK_UPDATES" + [ "$output" = "bash" ] + run env PATH="$bin:$PATH" bash --noprofile --norc -eo pipefail "$BATS_TEST_TMPDIR/step.sh" +} + +@test "check-updates: no line of the prepare run reaches the runner as a command" { + SUDO_OUT=$'::error::from a PKGBUILD\n::add-mask::failed' run_prepare_step + [ "$status" -eq 0 ] + [ "${#lines[@]}" -eq 4 ] + for line in "${lines[@]}"; do + [[ "$line" != "::"* ]] + done +} + +@test "check-updates: a failing prepare run fails its step" { + SUDO_OUT="boom" SUDO_RC=1 run_prepare_step + [ "$status" -ne 0 ] +} + +@test "check-updates: publish runs on the check job's decision, also without prepare" { + run yq '.jobs.publish.needs | join(" ")' "$CHECK_UPDATES" + [[ " $output " == *" check "* ]] + run yq '.jobs.publish.if' "$CHECK_UPDATES" + [[ "$output" == *"!cancelled()"* ]] + [[ "$output" == *"needs.check.outputs.work == 'true'"* ]] + [[ "$output" == *"needs.prepare.result"* ]] + [[ "$output" == *"needs.verify.result"* ]] +} + +@test "check-updates: publish takes the statuses from the check job" { + run yq '.jobs.publish.steps[] | select(.with.name == "checked") | .with.path' "$CHECK_UPDATES" + [ "$output" = ".status" ] + run yq '.jobs.publish.steps[] | select(.with.name == "verified") | .if' "$CHECK_UPDATES" + [ "$output" = "needs.verify.result == 'success'" ] +}