diff --git a/.github/skills/write-tests/SKILL.md b/.github/skills/write-tests/SKILL.md new file mode 100644 index 0000000..c154409 --- /dev/null +++ b/.github/skills/write-tests/SKILL.md @@ -0,0 +1,36 @@ +--- +name: write-tests +description: Write meaningful Replica tests. Use when adding regression coverage, testing new behavior, or changing existing automated tests. +--- + +# Writing meaningful tests + +## Establish the contract + +Read the [root instructions](../../../AGENTS.md), [shared coding conventions](../../../docs/coding-conventions.md), and [testing and review guide](../../../docs/testing-and-review.md), then the affected project's instructions and relevant guides. The shared guide owns test-quality criteria and the independent-writer policy. + +Identify the accepted behavior, public interface, and existing coverage. For each proposed case, state the distinct regression it catches and why existing cases would miss it. Drop redundant cases and incidental assertions. Complete this step when every retained case has an expected outcome grounded in requirements rather than current implementation. + +## Choose the writer + +Apply the shared guide's independent-writer policy. For substantial behavior changes and nontrivial bug fixes, delegate once to an available code-capable agent with permission to edit tests. Use a synchronous task unless implementation can safely proceed in separate files while the writer works. + +If you are already the assigned test writer, write the tests directly; this assignment is the independent pass. + +Give the writer a self-contained assignment containing: + +- The relevant user request and accepted requirements, separately from implementation suggestions. +- The bug reproduction or behavior to add, including known failure and boundary cases. +- Affected interfaces and paths, existing coverage, project guidance, and any implementation already completed. +- Ownership of named test files and necessary test fixtures; production changes remain with the implementation agent. +- A bounded result: implement the smallest distinct coverage, run focused checks, and return changed files, the regression each case catches, observed red/green evidence, and unresolved requirements or production defects. + +When behavior is unclear, resolve it with the user before asking the writer to encode it. Small changes follow the same process directly without delegation. + +## Write and verify + +Use the existing runner and repository test patterns. Establish a meaningful failing case where feasible, then run the smallest command covering the completed change. Preserve unrelated work when checking pre-fix behavior; prefer an isolated reproduction over reverting a dirty worktree. + +Review the resulting tests against the shared criteria. Ensure mocks leave the behavior under test real, expectations come from the contract, and every added case protects a distinct regression. Resolve reported production defects in the implementation agent, then rerun affected tests and follow the project's final-check requirements. + +Finish only when the retained cases prove their intended outcomes, or explicitly report the blocker and unverified behavior. Distinguish observed results from expected coverage, and maintenance simplification from measured runtime savings. Keep the handoff concise; the per-case rationale belongs in the writer's working result, not boilerplate code comments or a new tracking document. diff --git a/.github/workflows/actions-ci.yml b/.github/workflows/actions-ci.yml index 083e795..8274493 100644 --- a/.github/workflows/actions-ci.yml +++ b/.github/workflows/actions-ci.yml @@ -9,8 +9,13 @@ permissions: contents: read jobs: + changes: + uses: ./.github/workflows/ci-changes.yml + test: - name: Replica action tests + name: Build action suite + needs: changes + if: ${{ needs.changes.outputs.build_action == 'true' }} runs-on: ubuntu-latest steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 @@ -28,3 +33,22 @@ jobs: - name: Prepare composite action fixture run: cp actions/build/test/fixture/package.json actions/build/test/fixture/package-lock.json actions/build/test/fixture/fixture-build.mjs . - uses: ./actions/build + + result: + name: Replica action tests + needs: [changes, test] + if: ${{ always() }} + runs-on: ubuntu-latest + steps: + - name: Require the selected build-action suite to pass + env: + SELECTION_RESULT: ${{ needs.changes.result }} + SELECTED: ${{ needs.changes.outputs.build_action }} + TEST_RESULT: ${{ needs.test.result }} + run: | + test "$SELECTION_RESULT" = success + case "$SELECTED" in + true) test "$TEST_RESULT" = success ;; + false) test "$TEST_RESULT" = skipped ;; + *) echo "Invalid build-action suite selection" >&2; exit 1 ;; + esac diff --git a/.github/workflows/ci-changes.yml b/.github/workflows/ci-changes.yml new file mode 100644 index 0000000..0d160b3 --- /dev/null +++ b/.github/workflows/ci-changes.yml @@ -0,0 +1,40 @@ +name: CI selection + +on: + workflow_call: + outputs: + hub: + description: Run the complete Hub suite + value: ${{ jobs.changes.outputs.hub }} + build_action: + description: Run the build-action suite + value: ${{ jobs.changes.outputs.build_action }} + +permissions: + contents: read + +jobs: + changes: + name: Select affected suites + runs-on: ubuntu-latest + outputs: + hub: ${{ steps.select.outputs.hub }} + build_action: ${{ steps.select.outputs.build_action }} + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + fetch-depth: 0 + - uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0 + with: + node-version: 24 + - uses: ruby/setup-ruby@14594264cd68ce8a2345dd349bc3d138a4ef85c8 # v1.327.0 + with: + working-directory: hub + - run: node --test scripts/select-ci-suites.test.mjs + - run: ruby scripts/check-ci-workflows.rb + - id: select + run: node scripts/select-ci-suites.mjs + env: + CI_EVENT_NAME: ${{ github.event_name }} + CI_BASE_SHA: ${{ github.event.pull_request.base.sha }} + CI_HEAD_SHA: ${{ github.event.pull_request.head.sha }} diff --git a/.github/workflows/hub-ci.yml b/.github/workflows/hub-ci.yml index 51767c6..385f087 100644 --- a/.github/workflows/hub-ci.yml +++ b/.github/workflows/hub-ci.yml @@ -13,8 +13,13 @@ defaults: working-directory: hub jobs: + changes: + uses: ./.github/workflows/ci-changes.yml + rails_test: name: Rails tests + needs: changes + if: ${{ needs.changes.outputs.hub == 'true' }} runs-on: ubuntu-latest steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 @@ -45,6 +50,8 @@ jobs: development_queue: name: Development queue + needs: changes + if: ${{ needs.changes.outputs.hub == 'true' }} runs-on: ubuntu-latest steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 @@ -57,6 +64,8 @@ jobs: ruby_lint: name: Ruby lint + needs: changes + if: ${{ needs.changes.outputs.hub == 'true' }} runs-on: ubuntu-latest env: RUBOCOP_CACHE_ROOT: tmp/rubocop @@ -70,6 +79,8 @@ jobs: zeitwerk: name: Zeitwerk + needs: changes + if: ${{ needs.changes.outputs.hub == 'true' }} runs-on: ubuntu-latest steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 @@ -82,6 +93,8 @@ jobs: annotations: name: Annotations + needs: changes + if: ${{ needs.changes.outputs.hub == 'true' }} runs-on: ubuntu-latest steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 @@ -94,6 +107,8 @@ jobs: bundler_audit: name: Bundler audit + needs: changes + if: ${{ needs.changes.outputs.hub == 'true' }} runs-on: ubuntu-latest steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 @@ -105,6 +120,8 @@ jobs: typecheck: name: TypeScript types + needs: changes + if: ${{ needs.changes.outputs.hub == 'true' }} runs-on: ubuntu-latest steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 @@ -118,6 +135,8 @@ jobs: frontend_lint: name: Frontend lint + needs: changes + if: ${{ needs.changes.outputs.hub == 'true' }} runs-on: ubuntu-latest steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 @@ -131,23 +150,42 @@ jobs: frontend_test_shard: name: Frontend tests shard ${{ matrix.shard }}/4 + needs: changes + if: ${{ always() }} runs-on: ubuntu-latest strategy: fail-fast: false matrix: shard: [1, 2, 3, 4] steps: + - name: Require valid suite selection + working-directory: . + env: + SELECTION_RESULT: ${{ needs.changes.result }} + SELECTED: ${{ needs.changes.outputs.hub }} + run: | + test "$SELECTION_RESULT" = success + case "$SELECTED" in + true|false) ;; + *) echo "Invalid Hub suite selection" >&2; exit 1 ;; + esac - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + if: ${{ needs.changes.outputs.hub == 'true' }} - uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0 + if: ${{ needs.changes.outputs.hub == 'true' }} with: node-version-file: hub/package.json cache: npm cache-dependency-path: hub/package-lock.json - run: npm ci + if: ${{ needs.changes.outputs.hub == 'true' }} - run: npm test -- --shard=${{ matrix.shard }}/4 + if: ${{ needs.changes.outputs.hub == 'true' }} frontend_integration_test: name: Controls and extension tests + needs: changes + if: ${{ needs.changes.outputs.hub == 'true' }} runs-on: ubuntu-latest steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 @@ -169,19 +207,29 @@ jobs: frontend_test: name: Frontend tests - needs: [frontend_test_shard, frontend_integration_test] + needs: [changes, frontend_test_shard, frontend_integration_test] if: ${{ always() }} runs-on: ubuntu-latest steps: - name: Require all frontend suites to pass working-directory: . env: + SELECTION_RESULT: ${{ needs.changes.result }} + SELECTED: ${{ needs.changes.outputs.hub }} SHARD_RESULT: ${{ needs.frontend_test_shard.result }} INTEGRATION_RESULT: ${{ needs.frontend_integration_test.result }} - run: test "$SHARD_RESULT" = success && test "$INTEGRATION_RESULT" = success + run: | + test "$SELECTION_RESULT" = success + case "$SELECTED" in + true) test "$SHARD_RESULT" = success && test "$INTEGRATION_RESULT" = success ;; + false) test "$SHARD_RESULT" = success && test "$INTEGRATION_RESULT" = skipped ;; + *) echo "Invalid Hub suite selection" >&2; exit 1 ;; + esac frontend_build: name: Frontend build + needs: changes + if: ${{ needs.changes.outputs.hub == 'true' }} runs-on: ubuntu-latest steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 @@ -197,6 +245,8 @@ jobs: storybook_build: name: Storybook build + needs: changes + if: ${{ needs.changes.outputs.hub == 'true' }} runs-on: ubuntu-latest steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 @@ -210,6 +260,8 @@ jobs: production_database: name: Production database + needs: changes + if: ${{ needs.changes.outputs.hub == 'true' }} runs-on: ubuntu-latest steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 @@ -224,6 +276,8 @@ jobs: production_image: name: Production image + needs: changes + if: ${{ needs.changes.outputs.hub == 'true' }} runs-on: ubuntu-latest steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 diff --git a/AGENTS.md b/AGENTS.md index 64edc21..5411472 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -9,6 +9,7 @@ Read [Contributing](CONTRIBUTING.md) and the [shared documentation index](docs/R - [Coding conventions](docs/coding-conventions.md) apply to complete changes, tests, and comments. - [Documentation conventions](docs/documentation-conventions.md) apply to discussions, prose, labels, and errors. - [Testing and review](docs/testing-and-review.md) explains how to choose checks and review a change. +- Use the [test-writing skill](.github/skills/write-tests/SKILL.md) when adding or changing automated tests. Keep shared rules in root guides and project-specific rules in the affected project. Do not apply Hub's Rails, Primer, or devcontainer requirements to the starter, build action, or shared packages. The starter's copied instructions must remain usable without this monorepo. diff --git a/docs/testing-and-review.md b/docs/testing-and-review.md index 2896237..cd8bbcb 100644 --- a/docs/testing-and-review.md +++ b/docs/testing-and-review.md @@ -12,12 +12,54 @@ Use existing commands and inspect their configuration. Do not invent commands or For rendered checks and interaction reviews, follow the [root browser workflow](../AGENTS.md#browser-validation-and-playwright-fallback). Existing automated suites, CI browser integration, and build-artifact capture retain their configured drivers. +## Write meaningful tests + +Use the [test-writing skill](../.github/skills/write-tests/SKILL.md) when adding or changing automated tests. Apply these criteria when reviewing existing coverage too. + +Start with the intended behavior and the regression a test should catch. Inspect existing coverage before adding cases. Each case should protect a distinct outcome, boundary, or failure mode through the smallest useful public interface. An existing case that already catches the regression usually needs an update rather than another test. + +Choose assertions that survive an equivalent implementation. Check outcomes such as saved state, returned data, navigation, access decisions, and usable controls. Exact protocol values, serialization, accessible names, meaningful state mappings, and safety restrictions can be contracts worth asserting. Ordinary copy, icon counts, internal markup, and whole configuration snapshots usually are not. A text query that identifies a control is different from a test whose only purpose is freezing its wording. For agent instructions, distinguish discoverable metadata and required safety guidance from incidental phrasing; phrase presence alone does not prove agent behavior. + +Keep each behavior at its owning layer and use integration tests to prove important connections. Repeating all lower-level cases through every consumer adds cost without necessarily adding confidence. Keep separate cases for distinct failures; combine assertions when they describe one coherent scenario. Parameterization improves readability but still runs every input. Fewer case names or assertions do not establish a faster suite. + +For a bug fix, reproduce the failure before the fix when feasible. For new behavior, establish an expected-result failure before implementation when the interface allows it. A missing dependency, syntax error, or test setup failure is not evidence that the test catches the regression. If the implementation already exists, explain the plausible incorrect behavior the test would reject; report red-phase evidence as unavailable unless it was actually observed. + +### Independent test writing + +Use one separate test-writing agent for substantial behavior changes and nontrivial bug fixes. These include changes spanning multiple runtime layers, access or persistence rules, concurrent state changes, or a regression requiring non-obvious reproduction. Keep small, local changes in the implementation agent; a separate agent's context and execution cost should earn distinct coverage. + +Give the test writer the accepted requirements, relevant issue or bug reproduction, public interfaces, affected paths, existing coverage, and project instructions. Identify unresolved behavior instead of letting the writer infer requirements from current output. The writer should derive expected outcomes from that contract, then inspect implementation as needed to use the real interfaces. Separate context reduces shared assumptions; it does not guarantee independent reasoning. + +Assign test files to the writer and production files to the implementation agent. The writer may update test fixtures needed for its cases, but reports production defects and ambiguous requirements instead of changing production behavior or weakening expectations to pass. Finish with one integration review of the combined change. If a separate agent is unavailable, apply the same contract-first process directly and disclose that the independent pass did not happen. + ## Keep local checks and CI aligned Project commands define the check. GitHub Actions should prepare the required environment and call the same command rather than recreate its behavior. When a command gains a dependency or build prerequisite, update its local setup and affected workflow in the same change. A green local suite does not prove that CI installed the same dependencies, so inspect both paths. +### Select affected CI suites + +Hub and build-action workflows start on every pull request, but run their expensive jobs only when the change can affect that suite. Pushes to `main` run both complete suites. Local project commands still run their complete checks; selection changes when CI calls them, not what they do. Dependency-age checks remain unconditional, and Starter and Learning CI retain their own path filters. + +The [shared selection workflow](../.github/workflows/ci-changes.yml) calls [the selector](../scripts/select-ci-suites.mjs). For pull requests, it compares the merge base with the PR head using Git rather than the size-limited changed-files API. Deletions and both sides of renames count as changes. + +| Changed area | Selected suites | +| --- | --- | +| Hub code and configuration | Hub | +| Shared Controls runtime, schema, fixtures, and capture adapter under `hub/app/javascript/utils/prototype-controls/` | Hub and build action | +| `actions/build/` | Hub and build action; Hub uses the action for local packaging and browser checks | +| Copilot extension and plugin catalog | Hub | +| `packages/learning/` | Hub; its image, lint, and learning integration consume the package | +| `starter/` | Hub; local setup copies the starter and depends on its router, metadata, dependencies, and setup files; the Controls runtime suite also checks its shared reference. Starter CI still runs separately | +| Existing installer scripts | Neither; Starter CI covers them | +| Root contributor documents, shared and Hub guides, skills, and documentation assets | Neither | +| Workflow changes, selector changes, and unclassified paths | Both | + +The workflows preserve the existing required check names; skipping an entire workflow with a path filter could leave those checks pending. Static jobs use job-level conditions. The four required Vitest shard jobs always expand their matrix and validate selection, but skip checkout, setup, and tests when Hub is unaffected. The required `Frontend tests` and `Replica action tests` result jobs accept omitted work only after successful selection explicitly excluded it. Selection failures, invalid outputs, and failures in selected suites fail those result checks. Keep required-check configuration unchanged. + +When adding a shared dependency or a new project, update the selector and its behavioral tests together. Unclassified paths deliberately run both suites until their dependencies are known. Run `node --test scripts/select-ci-suites.test.mjs` and `ruby scripts/check-ci-workflows.rb` from the repository root. The latter reads the actual workflow gates and checks their success, failure, cancellation, and missing-output behavior; both commands run before CI selects suites. Check changed workflows with `actionlint` when available. Actual job skipping and check reporting still need verification on GitHub after publishing. + ## Update dependencies Hub and the shared build action wait seven days before resolving newly published dependencies. npm enforces the delay through each project's committed `.npmrc`. Hub's RubyGems source in `Gemfile` applies the delay to Bundler. Dependabot applies the same cooldown to npm, Bundler, and GitHub Actions updates across the repository. The starter manages its dependency resolution independently. diff --git a/hub/app/javascript/pages/welcome-page/index.test.tsx b/hub/app/javascript/pages/welcome-page/index.test.tsx index 544217e..428a3b8 100644 --- a/hub/app/javascript/pages/welcome-page/index.test.tsx +++ b/hub/app/javascript/pages/welcome-page/index.test.tsx @@ -76,15 +76,6 @@ describe("WelcomePage()", () => { "data-sg-visit", "true", ) - expect( - screen.getByRole("link", { name: data.readmeLabel }).querySelectorAll("svg"), - ).toHaveLength(2) - expect( - screen.getByRole("link", { name: data.setupLabel }).querySelectorAll("svg"), - ).toHaveLength(2) - expect( - screen.getByRole("link", { name: data.createLabel }).querySelectorAll("svg"), - ).toHaveLength(1) for (const label of [data.readmeLabel, data.setupLabel, data.createLabel]) { expect(screen.getByRole("heading", { level: 2, name: label })).toBeInTheDocument() } diff --git a/hub/docs/testing-and-operations.md b/hub/docs/testing-and-operations.md index 6c565c5..3509a37 100644 --- a/hub/docs/testing-and-operations.md +++ b/hub/docs/testing-and-operations.md @@ -64,7 +64,7 @@ A Rails test job that renders Vite pages must: When a Hub command gains a new dependency or build prerequisite, update `config/ci.rb` and the [Hub CI workflow](../../.github/workflows/hub-ci.yml) in the same change. -GitHub CI splits `npm test` across four Vitest shards and runs controls and extension integration tests in a separate parallel job. The `Frontend tests` check succeeds only when every shard and the integration job pass. Local `bin/ci` still runs the complete suites without sharding. +When Hub is affected, GitHub CI splits `npm test` across four Vitest shards and runs controls and extension integration tests in a separate parallel job. The `Frontend tests` check requires every selected shard and integration job to pass. Unrelated pull requests omit that work through the [shared suite-selection policy](../../docs/testing-and-review.md#select-affected-ci-suites). Local `bin/ci` still runs the complete suites without sharding or selection. Vitest runs explicitly listed browser-independent tests in its `node` project without DOM setup. Other tests stay in the `browser` project with jsdom and shared browser cleanup. Both projects retain mock and global resets and test isolation. Add a test to the Node list in `vitest.config.ts` only after checking its imports and runtime dependencies; a `.ts` extension does not establish browser independence. `npm test` and path selectors cover both projects, while `npm test -- --project=node` selects the Node tests. diff --git a/scripts/check-ci-workflows.rb b/scripts/check-ci-workflows.rb new file mode 100644 index 0000000..fe3f0b6 --- /dev/null +++ b/scripts/check-ci-workflows.rb @@ -0,0 +1,94 @@ +require "yaml" +require "open3" + +def check(condition, message) + abort(message) unless condition +end + +def expression(value) + value.to_s.strip.delete_prefix("${{").delete_suffix("}}").strip +end + +root = File.expand_path("..", __dir__) +workflows = %w[hub-ci actions-ci].to_h do |name| + [name, YAML.safe_load(File.read(File.join(root, ".github/workflows/#{name}.yml"))).fetch("jobs")] +end +expected_gates = { + ["hub-ci", "frontend_test_shard"] => ["Frontend tests shard ${{ matrix.shard }}/4", ["changes"]], + ["hub-ci", "frontend_test"] => ["Frontend tests", %w[changes frontend_test_shard frontend_integration_test]], + ["actions-ci", "result"] => ["Replica action tests", %w[changes test]] +} +gates = {} + +workflows.each do |workflow, jobs| + suite = workflow == "hub-ci" ? "hub" : "build_action" + selection = "needs.changes.outputs.#{suite} == 'true'" + jobs.each do |id, job| + next if id == "changes" + + label = "#{workflow}/#{id}" + check(Array(job["needs"]).include?("changes"), "#{label}: missing changes dependency") + steps = job.fetch("steps") + guards = steps.select { |step| step.fetch("env", {}).key?("SELECTION_RESULT") } + if guards.empty? + check(expression(job["if"]) == selection, "#{label}: suite must require selection") + next + end + + key = [workflow, id] + check(expected_gates.key?(key), "#{label}: unexpected gate") + check(guards.length == 1, "#{label}: expected exactly one selection guard") + name, needs = expected_gates.fetch(key) + check(job["name"] == name, "#{label}: required check name changed") + check(Array(job["needs"]).sort == needs.sort, "#{label}: incorrect gate dependencies") + check(expression(job["if"]) == "always()", "#{label}: gate must always run") + guard = guards.first + check(!guard.key?("if"), "#{label}: guard must not be conditional") + env = guard.fetch("env") + check(expression(env["SELECTION_RESULT"]) == "needs.changes.result", "#{label}: incorrect selection result") + check(expression(env["SELECTED"]) == "needs.changes.outputs.#{suite}", "#{label}: incorrect selection output") + result_jobs = { "SHARD_RESULT" => "frontend_test_shard", "INTEGRATION_RESULT" => "frontend_integration_test", "TEST_RESULT" => "test" } + expected_results = id == "frontend_test" ? %w[SHARD_RESULT INTEGRATION_RESULT] : id == "result" ? ["TEST_RESULT"] : [] + check(env.keys.sort == (%w[SELECTION_RESULT SELECTED] + expected_results).sort, "#{label}: incorrect gate inputs") + expected_results.each do |variable| + check(expression(env[variable]) == "needs.#{result_jobs.fetch(variable)}.result", "#{label}: incorrect #{variable}") + end + if id == "frontend_test_shard" + check(job.dig("strategy", "matrix", "shard") == [1, 2, 3, 4], "#{label}: required shards changed") + check(steps.first.equal?(guard) && steps.length > 1, "#{label}: guard must precede suite steps") + check(steps.drop(1).all? { |step| expression(step["if"]) == selection }, "#{label}: suite steps must require selection") + end + gates[key] = guard.fetch("run") + end +end +check(gates.keys.sort == expected_gates.keys.sort, "Expected exactly the shard guard and both final result gates") +check(workflows.fetch("actions-ci").fetch("test")["name"] == "Build action suite", "Build action suite name changed") + +states = %w[success failure cancelled skipped] + [""] +checks = 0 +gates.each do |(workflow, id), script| + variables = case id + when "frontend_test" then %w[SHARD_RESULT INTEGRATION_RESULT] + when "result" then ["TEST_RESULT"] + else [] + end + combinations = variables.reduce([{}]) { |rows, variable| rows.flat_map { |row| states.map { |state| row.merge(variable => state) } } } + states.product(%w[true false invalid] + [""]).each do |selection_result, selected| + combinations.each do |results| + expected = selection_result == "success" && %w[true false].include?(selected) + expected &&= case id + when "frontend_test_shard" then true + when "frontend_test" + results["SHARD_RESULT"] == "success" && + results["INTEGRATION_RESULT"] == (selected == "true" ? "success" : "skipped") + when "result" + results["TEST_RESULT"] == (selected == "true" ? "success" : "skipped") + end + env = results.merge("SELECTION_RESULT" => selection_result, "SELECTED" => selected) + stdout, stderr, status = Open3.capture3(env.merge("BASH_ENV" => nil), "bash", "--noprofile", "--norc", "-e", "-c", script, chdir: root) + check(status.success? == expected, "#{workflow}/#{id} #{env.inspect}: expected #{expected ? 'pass' : 'failure'}, exit #{status.exitstatus}\n#{stdout}#{stderr}") + checks += 1 + end + end +end +puts "CI workflow guards and wiring passed (#{checks} shell cases)." diff --git a/scripts/create-prototype-repo.test.mjs b/scripts/create-prototype-repo.test.mjs index f8c3321..5983471 100644 --- a/scripts/create-prototype-repo.test.mjs +++ b/scripts/create-prototype-repo.test.mjs @@ -413,8 +413,6 @@ test("creates an independent repository with exactly the tracked starter files", assert.equal(execFileSync("git", ["remote"], { cwd: destination }).toString(), "") assert.equal(spawnSync("git", ["rev-parse", "HEAD"], { cwd: destination }).status, 128) assert.equal(actual.length, paths.length) - assertContextBundle(destination) - assertBrowserSetup(destination) assert.ok(!readdirSync(directory).some(name => name.startsWith(".replica-starter."))) }) @@ -446,8 +444,6 @@ test("installs a public-style archive without a checkout or credentials", t => { ) assert.equal(execFileSync("git", ["remote"], { cwd: destination }).toString(), "") assert.equal(spawnSync("git", ["rev-parse", "HEAD"], { cwd: destination }).status, 128) - assertContextBundle(destination) - assertBrowserSetup(destination) assertNoTemporaryFiles(fixture.directory) }) diff --git a/scripts/select-ci-suites.mjs b/scripts/select-ci-suites.mjs new file mode 100644 index 0000000..4614393 --- /dev/null +++ b/scripts/select-ci-suites.mjs @@ -0,0 +1,89 @@ +import { execFileSync } from "node:child_process" +import { appendFileSync } from "node:fs" +import { pathToFileURL } from "node:url" + +const documentation = new Set([ + "AGENTS.md", + "CODE_OF_CONDUCT.md", + "CONTRIBUTING.md", + "LICENSE", + "NOTICE.md", + "README.md", + "SECURITY.md", + "SUPPORT.md", + ".github/pull_request_template.md", +]) + +const starterScripts = new Set([ + "scripts/create-prototype-repo.sh", + "scripts/create-prototype-repo.test.mjs", + "scripts/install-starter.sh", + "scripts/setup-replica-plugin.test.mjs", +]) + +export function selectCiSuites(paths) { + const suites = { hub: false, build_action: false } + for (const path of paths) { + if (path.startsWith("hub/app/javascript/utils/prototype-controls/")) { + suites.hub = true + suites.build_action = true + } else if (path.startsWith("actions/build/")) { + suites.hub = true + suites.build_action = true + } else if ( + documentation.has(path) || + path.startsWith("docs/") || + path.startsWith("hub/docs/") || + path === "hub/AGENTS.md" || + path.startsWith(".github/skills/") || + path.startsWith(".github/assets/") || + path.startsWith(".github/ISSUE_TEMPLATE/") + ) { + continue + } else if ( + path.startsWith("hub/") || + path.startsWith(".github/extensions/replica-prototype/") || + path.startsWith(".github/plugin/") || + path.startsWith("packages/learning/") || + path.startsWith("starter/") + ) { + suites.hub = true + } else if (starterScripts.has(path)) { + continue + } else { + suites.hub = true + suites.build_action = true + } + } + return suites +} + +export function changedCiSuites(event, base, head) { + if (event === "push") return { hub: true, build_action: true } + if (event !== "pull_request") throw new Error(`Unsupported CI event: ${event}`) + if (![base, head].every((sha) => typeof sha === "string" && /^[a-f0-9]{40}$/.test(sha))) { + throw new Error("CI selection requires full base and head commit SHAs.") + } + const paths = execFileSync( + "git", + ["diff", "--name-only", "--no-renames", "-z", `${base}...${head}`, "--"], + { encoding: "utf8", maxBuffer: 32 * 1024 * 1024 }, + ) + .split("\0") + .filter(Boolean) + return selectCiSuites(paths) +} + +if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) { + const suites = changedCiSuites( + process.env.CI_EVENT_NAME, + process.env.CI_BASE_SHA, + process.env.CI_HEAD_SHA, + ) + const output = Object.entries(suites) + .map(([suite, selected]) => `${suite}=${selected}\n`) + .join("") + if (!process.env.GITHUB_OUTPUT) throw new Error("GITHUB_OUTPUT is required for CI selection.") + appendFileSync(process.env.GITHUB_OUTPUT, output) + process.stdout.write(output) +} diff --git a/scripts/select-ci-suites.test.mjs b/scripts/select-ci-suites.test.mjs new file mode 100644 index 0000000..d70e5fe --- /dev/null +++ b/scripts/select-ci-suites.test.mjs @@ -0,0 +1,262 @@ +import assert from "node:assert/strict" +import { spawnSync } from "node:child_process" +import { + mkdirSync, + mkdtempSync, + readFileSync, + rmSync, + unlinkSync, + writeFileSync, +} from "node:fs" +import { dirname, join } from "node:path" +import { after, test } from "node:test" +import { fileURLToPath, pathToFileURL } from "node:url" +import { selectCiSuites } from "./select-ci-suites.mjs" + +const root = dirname(dirname(fileURLToPath(import.meta.url))) +const script = join(root, "scripts/select-ci-suites.mjs") +const selectorUrl = pathToFileURL(script).href +const fixtureRoot = mkdtempSync(join(root, ".select-ci-suites-test-")) +after(() => rmSync(fixtureRoot, { recursive: true, force: true })) + +function git(repo, ...args) { + const result = spawnSync("git", args, { cwd: repo, encoding: "utf8" }) + assert.ifError(result.error) + assert.equal(result.status, 0, result.stderr) + return result.stdout.trim() +} + +function createRepo(name) { + const repo = join(fixtureRoot, name) + mkdirSync(repo) + git(repo, "init", "--quiet") + git(repo, "config", "user.name", "CI suite test") + git(repo, "config", "user.email", "ci-suite-test@example.invalid") + git(repo, "config", "commit.gpgsign", "false") + writeFileSync(join(repo, "README.md"), "fixture\n") + git(repo, "add", "--all") + git(repo, "commit", "--quiet", "-m", "initial") + return repo +} + +function writeFile(repo, path, content = "fixture\n") { + const file = join(repo, path) + mkdirSync(dirname(file), { recursive: true }) + writeFileSync(file, content) +} + +function commit(repo, message) { + git(repo, "add", "--all") + git(repo, "commit", "--quiet", "-m", message) + return git(repo, "rev-parse", "HEAD") +} + +function runCli(repo, env) { + return spawnSync(process.execPath, [script], { + cwd: repo, + encoding: "utf8", + env: { ...process.env, ...env }, + timeout: 10_000, + }) +} + +function runChangedCiSuites(repo, args) { + const source = ` + import { changedCiSuites } from ${JSON.stringify(selectorUrl)} + const args = JSON.parse(process.env.SELECTOR_ARGS) + try { + process.stdout.write(JSON.stringify(changedCiSuites(...args))) + } catch (error) { + console.error(error.message) + process.exitCode = 1 + } + ` + const result = spawnSync(process.execPath, ["--input-type=module", "-e", source], { + cwd: repo, + encoding: "utf8", + env: { ...process.env, SELECTOR_ARGS: JSON.stringify(args) }, + timeout: 10_000, + }) + assert.ifError(result.error) + return result +} + +function selectChangedFiles(repo, ...args) { + const result = runChangedCiSuites(repo, args) + assert.equal(result.status, 0, result.stderr) + return JSON.parse(result.stdout) +} + +test("path selection keeps documentation and installer scripts out while covering Hub consumers", () => { + const excluded = [ + "README.md", + "docs/guide.md", + "hub/docs/guide.md", + "hub/AGENTS.md", + ".github/skills/write-tests/SKILL.md", + ".github/assets/logo.svg", + ".github/ISSUE_TEMPLATE/bug.yml", + "scripts/create-prototype-repo.sh", + "scripts/create-prototype-repo.test.mjs", + "scripts/install-starter.sh", + "scripts/setup-replica-plugin.test.mjs", + ] + for (const path of excluded) { + assert.deepEqual(selectCiSuites([path]), { hub: false, build_action: false }, path) + } + + for (const path of [ + "hub/app/models/prototype.rb", + ".github/extensions/replica-prototype/extension.mjs", + ".github/plugin/README.md", + "packages/learning/src/index.mjs", + "starter/.agents/skills/prototype-controls/SKILL.md", + "starter/src/app/router.tsx", + "starter/package-lock.json", + "starter/.prototype", + "starter/.agents/skills/setup-skills/SKILL.md", + "starter/src/App.jsx", + ]) { + assert.deepEqual(selectCiSuites([path]), { hub: true, build_action: false }, path) + } + for (const path of [ + "hub/app/javascript/utils/prototype-controls/controls.ts", + "actions/build/src/index.mjs", + ]) { + assert.deepEqual(selectCiSuites([path]), { hub: true, build_action: true }, path) + } +}) + +test("mixed changes union selections and unknown or CI configuration paths select both suites", () => { + assert.deepEqual( + selectCiSuites(["docs/guide.md", "hub/app/models/prototype.rb", "starter/src/App.jsx"]), + { hub: true, build_action: false }, + ) + for (const path of [ + ".github/workflows/hub-ci.yml", + ".github/workflows/ci-changes.yml", + "scripts/select-ci-suites.mjs", + ".github/extensions/new-extension/extension.yml", + "new-root-file.txt", + ]) { + assert.deepEqual(selectCiSuites([path]), { hub: true, build_action: true }, path) + } +}) + +test("pushes always select both suites while pull requests diff from their merge base", () => { + const repo = createRepo("merge-base") + assert.deepEqual(selectChangedFiles(repo, "push"), { hub: true, build_action: true }) + const common = git(repo, "rev-parse", "HEAD") + git(repo, "switch", "--quiet", "-c", "base") + writeFile(repo, "hub/base-only.rb") + const base = commit(repo, "change base") + git(repo, "switch", "--quiet", "-c", "head", common) + writeFile(repo, "docs/release\nhub/not-a-path.rb") + writeFile(repo, "docs/guide.md") + const head = commit(repo, "change pull request") + + assert.deepEqual(selectChangedFiles(repo, "pull_request", base, head), { + hub: false, + build_action: false, + }) +}) + +test("pull-request diffs count deleted paths and both sides of cross-boundary renames", () => { + const deletionRepo = createRepo("deletion") + writeFile(deletionRepo, "actions/build/removed.mjs") + const withFile = commit(deletionRepo, "add build file") + unlinkSync(join(deletionRepo, "actions/build/removed.mjs")) + const deletionHead = commit(deletionRepo, "delete build file") + assert.deepEqual(selectChangedFiles(deletionRepo, "pull_request", withFile, deletionHead), { + hub: true, + build_action: true, + }) + + const oldPathRepo = createRepo("rename-old-path") + writeFile(oldPathRepo, "actions/build/renamed.mjs") + const oldPathWithFile = commit(oldPathRepo, "add action file") + mkdirSync(join(oldPathRepo, "docs"), { recursive: true }) + git(oldPathRepo, "mv", "actions/build/renamed.mjs", "docs/renamed.mjs") + const oldPathHead = commit(oldPathRepo, "move action file to docs") + assert.deepEqual(selectChangedFiles(oldPathRepo, "pull_request", oldPathWithFile, oldPathHead), { + hub: true, + build_action: true, + }) + + const newPathRepo = createRepo("rename-new-path") + writeFile(newPathRepo, "docs/renamed.mjs") + const newPathWithFile = commit(newPathRepo, "add documentation file") + mkdirSync(join(newPathRepo, "hub/app"), { recursive: true }) + git(newPathRepo, "mv", "docs/renamed.mjs", "hub/app/renamed.mjs") + const newPathHead = commit(newPathRepo, "move docs to Hub") + assert.deepEqual(selectChangedFiles(newPathRepo, "pull_request", newPathWithFile, newPathHead), { + hub: true, + build_action: false, + }) +}) + +test("invalid event data and unavailable git history fail explicitly", () => { + const repo = createRepo("invalid-data") + const unsupported = runChangedCiSuites(repo, ["schedule"]) + assert.notEqual(unsupported.status, 0) + assert.match(unsupported.stderr, /Unsupported CI event/) + const validSha = "b".repeat(40) + for (const [base, head] of [ + [undefined, validSha], + [validSha, undefined], + ["abc", validSha], + [validSha, "abc"], + ["A".repeat(40), validSha], + [validSha, "A".repeat(40)], + ]) { + const result = runChangedCiSuites(repo, ["pull_request", base, head]) + assert.notEqual(result.status, 0) + assert.match(result.stderr, /full base and head commit SHAs/) + } + + const missingHistory = runChangedCiSuites(repo, [ + "pull_request", + "a".repeat(40), + "b".repeat(40), + ]) + assert.notEqual(missingHistory.status, 0) + assert.match(missingHistory.stderr, /fatal|git/i) +}) + +test("the CLI writes suite booleans and fails without an output file", () => { + const repo = createRepo("cli") + const base = git(repo, "rev-parse", "HEAD") + writeFile(repo, "hub/app/models/prototype.rb") + const head = commit(repo, "change Hub") + const output = join(repo, "github-output") + const env = { + CI_EVENT_NAME: "pull_request", + CI_BASE_SHA: base, + CI_HEAD_SHA: head, + GITHUB_OUTPUT: output, + } + + const result = runCli(repo, env) + assert.ifError(result.error) + assert.equal(result.status, 0, result.stderr) + assert.deepEqual(result.stdout.trim().split("\n").sort(), [ + "build_action=false", + "hub=true", + ]) + assert.deepEqual(readFileSync(output, "utf8").trim().split("\n").sort(), [ + "build_action=false", + "hub=true", + ]) + + const missingOutputEnv = { ...env, GITHUB_OUTPUT: undefined } + const missingOutput = runCli(repo, missingOutputEnv) + assert.ifError(missingOutput.error) + assert.notEqual(missingOutput.status, 0) + assert.equal(missingOutput.stdout, "") + assert.match(missingOutput.stderr, /GITHUB_OUTPUT is required/) + assert.equal(missingOutput.signal, null) + assert.deepEqual(readFileSync(output, "utf8").trim().split("\n").sort(), [ + "build_action=false", + "hub=true", + ]) +})