ci(test): packed layer in a fresh app and against comark.dev

.github/actions/setup/action.yml

@@ -0,0 +1,20 @@
+name: setup
+description: Node.js and the pnpm version `packageManager` pins, then the workspace dependencies.
+
+inputs:
+  install:
+    description: Install the workspace dependencies, not needed by jobs that only pack the layer.
+    default: 'true'
+
+runs:
+  using: composite
+  steps:
+    - uses: actions/setup-node@v6
+      with:
+        package-manager-cache: false
+        node-version: '24'
+    - run: npm i -g --force corepack && corepack enable
+      shell: bash
+    - if: inputs.install == 'true'
+      run: pnpm install --frozen-lockfile
+      shell: bash

.github/workflows/ci.yml

@@ -22,12 +22,7 @@ jobs:
 
     steps:
       - uses: actions/checkout@v6
-      - uses: actions/setup-node@v6
-        with:
-          package-manager-cache: false
-          node-version: '24'
-      - run: npm i -g --force corepack && corepack enable
-      - run: pnpm install --frozen-lockfile
+      - uses: ./.github/actions/setup
       # Lint and unit tests need no generated types, so they run before `prepare`
       # and fail fast.
       - run: pnpm lint
@@ -51,14 +46,36 @@ jobs:
 
     steps:
       - uses: actions/checkout@v6
-      - uses: actions/setup-node@v6
-        with:
-          package-manager-cache: false
-          node-version: '24'
-      - run: npm i -g --force corepack && corepack enable
-      - run: pnpm install --frozen-lockfile
+      - uses: ./.github/actions/setup
       - run: pnpm build
         env:
           NITRO_PRESET: vercel
           VERCEL_BYPASS_TOKEN: ci-placeholder-bypass-token-0000000000
       - run: pnpm test:vercel
+
+  # The layer as a site installs it: packed, outside the workspace, built in a fresh app.
+  pack:
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v6
+      - uses: ./.github/actions/setup
+        with:
+          install: false
+      - run: ./scripts/test-pack.sh
+
+  # comark.dev, a site extending the layer, built against this commit instead of the version it pins.
+  # cms.comark.dev joins once its repository is public, a private one would need a token secret.
+  consumer:
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v6
+      - uses: actions/checkout@v6
+        with:
+          repository: comarkdown/comark
+          path: .consumer
+      - uses: ./.github/actions/setup
+        with:
+          install: false
+      - run: ./scripts/test-consumer.sh .consumer

.github/workflows/smoke.yml

@@ -0,0 +1,35 @@
+name: smoke
+
+# Vercel's GitHub integration reports every deployment, preview and production, through this event.
+# Content-only pushes skip the build (Ignored Build Step), so they deploy nothing and run nothing here.
+on: deployment_status
+
+permissions:
+  contents: read
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.deployment.id }}
+  cancel-in-progress: true
+
+jobs:
+  smoke:
+    name: smoke (${{ github.event.deployment.environment }})
+    if: github.event.deployment_status.state == 'success'
+    runs-on: ubuntu-latest
+    timeout-minutes: 15
+
+    steps:
+      # The tests of the deployed commit, not the default branch's.
+      - uses: actions/checkout@v6
+        with:
+          ref: ${{ github.event.deployment.sha }}
+          persist-credentials: false
+      - uses: ./.github/actions/setup
+      - run: pnpm test:smoke
+        env:
+          SMOKE_URL: ${{ github.event.deployment_status.environment_url || github.event.deployment_status.target_url }}
+          SMOKE_ENVIRONMENT: ${{ github.event.deployment.environment }}
+          # Vercel project → Deployment Protection → Protection Bypass for Automation.
+          VERCEL_AUTOMATION_BYPASS_SECRET: ${{ secrets.VERCEL_AUTOMATION_BYPASS_SECRET }}
+          # Set the repository variable to `true` to also call the assistant, which costs AI Gateway calls.
+          SMOKE_ASSISTANT: ${{ vars.SMOKE_ASSISTANT }}

README.md

@@ -72,11 +72,13 @@ Each skill is a directory with a `SKILL.md` whose frontmatter includes a `descri
 
 ```bash
 pnpm install
-pnpm dev       # runs the playground
-pnpm lint      # correctness-only ESLint (no stylistic rules — see eslint.config.mjs)
-pnpm test      # Vitest over the pure utils
-pnpm test:e2e  # Vitest against the built playground (`pnpm build` first)
-pnpm typecheck # vue-tsc across the layer + playground
+pnpm dev         # runs the playground
+pnpm lint        # correctness-only ESLint (no stylistic rules — see eslint.config.mjs)
+pnpm test        # Vitest over the pure utils
+pnpm test:e2e    # Vitest against the built playground (`pnpm build` first)
+pnpm test:vercel # Vitest over the Vercel output (`NITRO_PRESET=vercel pnpm build` first)
+pnpm test:smoke  # the e2e suite against a deployment (`SMOKE_URL=https://… pnpm test:smoke`)
+pnpm typecheck   # vue-tsc across the layer + playground
 ```
 
 To develop the layer against a consumer app in a sibling checkout:

package.json

@@ -37,6 +37,7 @@
     "test": "vitest run --project unit",
     "test:e2e": "vitest run --project e2e",
     "test:vercel": "vitest run --project vercel",
+    "test:smoke": "vitest run --project smoke",
     "test:watch": "vitest --project unit",
     "dev:prepare": "nuxt prepare playground"
   },

scripts/test-consumer.sh

@@ -0,0 +1,29 @@
+#!/usr/bin/env bash
+# Builds a site extending the layer against the packed layer, instead of the version it pins.
+# Usage: scripts/test-consumer.sh <site-checkout> [app-dir]
+# - `app-dir` is the Nuxt app inside the site's repository, `docs` by default.
+set -euo pipefail
+
+site="$(cd "${1:?Usage: scripts/test-consumer.sh <site-checkout> [app-dir]}" && pwd)"
+app="${2:-docs}"
+root="$(cd "$(dirname "$0")/.." && pwd)"
+work="$(mktemp -d)"
+trap 'rm -rf "$work"' EXIT
+
+echo "› Packing the layer"
+pnpm --dir "$root" pack --pack-destination "$work" > /dev/null
+tarball="$(ls "$work"/comark-docs-*.tgz)"
+
+# A copy, so the checkout keeps its own layer version and lockfile.
+# `.git` stays: the layer reads the content commit from it.
+echo "› Copying the site to $work/site"
+rsync -a --exclude node_modules --exclude .nuxt --exclude .output --exclude .vercel --exclude .data "$site/" "$work/site/"
+cd "$work/site"
+
+echo "› Pointing $app at the packed layer"
+(cd "$app" && npm pkg set "dependencies.comark-docs=file:$tarball")
+# The site's lockfile pins the layer it ships, so it has to move.
+pnpm install --no-frozen-lockfile
+
+echo "› Building $app"
+pnpm --dir "$app" run build

scripts/test-pack.sh

@@ -0,0 +1,42 @@
+#!/usr/bin/env bash
+# Packs the layer as a consumer installs it, then builds a fresh copy of test/fixtures/consumer with it.
+# - Catches what the workspace hides: a file missing from `files`, a dependency only the workspace provides.
+# - `WORKDIR` keeps the app there instead of in a temporary folder.
+set -euo pipefail
+
+root="$(cd "$(dirname "$0")/.." && pwd)"
+
+if [ -n "${WORKDIR:-}" ]; then
+  work="$WORKDIR"
+  if [ -e "$work/app" ]; then
+    echo "$work/app already exists, remove it or pick another WORKDIR" >&2
+    exit 1
+  fi
+  mkdir -p "$work"
+else
+  work="$(mktemp -d)"
+  trap 'rm -rf "$work"' EXIT
+fi
+
+echo "› Packing the layer"
+pnpm --dir "$root" pack --pack-destination "$work" > /dev/null
+tarball="$(ls "$work"/comark-docs-*.tgz)"
+
+# Outside the repo: pnpm would otherwise pick up its workspace and overrides.
+echo "› Copying the consumer fixture to $work/app"
+cp -R "$root/test/fixtures/consumer" "$work/app"
+cd "$work/app"
+
+# A git checkout like a real site, so the build-time content snapshot runs too.
+# The empty root commit gives the content commit a parent, as in any non-shallow clone.
+git init -q
+git -c user.name=ci -c user.email=ci@localhost commit -q --allow-empty -m root
+git add -A
+git -c user.name=ci -c user.email=ci@localhost commit -qm fixture
+
+echo "› Installing the packed layer"
+npm pkg set "dependencies.comark-docs=file:$tarball" "packageManager=$(npm pkg get packageManager --prefix "$root" | tr -d '"')"
+pnpm install
+
+echo "› Building"
+pnpm exec nuxt build

test/e2e/mcp.test.ts

@@ -1,11 +1,11 @@
 import { describe, expect, it } from 'vitest'
-import { baseURL, fetchWithFailure, INTRO_PAGE, NOT_FOUND_SLUG, SITE_URL } from './utils'
+import { fetchWithFailure, INTRO_PAGE, NOT_FOUND_SLUG, request, SITE_URL } from './utils'
 
 let requestId = 0
 
 /** Calls the MCP server over streamable HTTP, without a session. */
 async function mcp<T>(method: string, params?: Record<string, unknown>): Promise<T> {
-  const response = await fetch(new URL('/mcp', baseURL), {
+  const response = await request('/mcp', {
     method: 'POST',
     headers: { 'content-type': 'application/json', accept: 'application/json, text/event-stream' },
     body: JSON.stringify({ jsonrpc: '2.0', id: ++requestId, method, params }),

test/e2e/utils.ts

@@ -14,9 +14,19 @@ export const INTRO_PAGE = { path: '/getting-started/introduction', title: 'Intro
 /** A slug no page uses. */
 export const NOT_FOUND_SLUG = '__e2e-not-found__'
 
+// Only set against a protected Vercel deployment, see `test/smoke/`.
+const bypassSecret = process.env.VERCEL_AUTOMATION_BYPASS_SECRET
+
+/** Requests a site path, passing Vercel's Deployment Protection when a bypass secret is set. */
+export function request(path: string, init: RequestInit = {}): Promise<Response> {
+  const headers = new Headers(init.headers)
+  if (bypassSecret) headers.set('x-vercel-protection-bypass', bypassSecret)
+  return fetch(new URL(path, baseURL), { ...init, headers })
+}
+
 /** Fetches a site path once: use it for expected errors, which the server logs on every request. */
 export function fetchPath(path: string, headers?: HeadersInit): Promise<Response> {
-  return fetch(new URL(path, baseURL), { headers })
+  return request(path, { headers })
 }
 
 /** Fetches a site path, explaining a failure with the message Nitro attaches to its JSON errors. */

test/fixtures/consumer/content/1.getting-started/.navigation.yml

@@ -0,0 +1 @@
+title: Getting Started

test/fixtures/consumer/content/1.getting-started/1.introduction.md

@@ -0,0 +1,6 @@
+---
+title: Introduction
+description: What this project does and why it exists.
+---
+
+Welcome to the docs!

test/fixtures/consumer/content/index.md

@@ -0,0 +1,9 @@
+---
+title: Consumer Fixture
+description: A site extending the packed comark-docs layer.
+navigation: false
+---
+
+# Consumer Fixture
+
+Read the [introduction](/getting-started/introduction).

test/fixtures/consumer/nuxt.config.ts

@@ -0,0 +1,8 @@
+// The setup the installation guide documents, nothing more.
+export default defineNuxtConfig({
+  extends: ['comark-docs'],
+  site: {
+    url: 'https://docs.example.com',
+    name: 'Consumer Fixture',
+  },
+})

test/fixtures/consumer/package.json

@@ -0,0 +1,9 @@
+{
+  "name": "comark-docs-consumer",
+  "private": true,
+  "type": "module",
+  "dependencies": {
+    "comark-docs": "set by scripts/test-pack.sh",
+    "nuxt": "^4.6.2"
+  }
+}

test/fixtures/consumer/pnpm-workspace.yaml

@@ -0,0 +1,15 @@
+# The layer depends on a pkg.pr.new build of nuxt-llms, which pnpm blocks in subdependencies by default.
+# Remove once nuxt-llms ships a registry release the layer can depend on instead.
+blockExoticSubdeps: false
+
+# pnpm fails a non-interactive install on unreviewed build scripts, same decisions as the repo root.
+allowBuilds:
+  esbuild: false
+  vue-demi: false
+
+minimumReleaseAgeExclude:
+  - '@comark/nuxt'
+  - '@comark/vue'
+  - comark
+  - comark-content
+  - nuxt-agent-discovery

test/smoke/setup.ts

@@ -0,0 +1,10 @@
+import type { TestProject } from 'vitest/node'
+
+/** Points the e2e suite at a deployment instead of booting the local build. */
+export default function setup(project: TestProject) {
+  const url = process.env.SMOKE_URL
+  if (!url) {
+    throw new Error('No deployment to test: set `SMOKE_URL`, e.g. `SMOKE_URL=https://docs-template.comark.dev pnpm test:smoke`.')
+  }
+  project.provide('baseURL', new URL(url).origin)
+}

test/smoke/vercel.test.ts

@@ -0,0 +1,50 @@
+import { setTimeout as sleep } from 'node:timers/promises'
+import { describe, expect, it } from 'vitest'
+import { INTRO_PAGE, request } from '../e2e/utils'
+
+const FROM_CACHE = /^(?:HIT|STALE|PRERENDER)$/
+
+describe('ISR', () => {
+  it('serves a repeat request from the cache', async () => {
+    let cache: string | null = null
+    // Vercel stores a fresh response about a second after serving it, so an immediate repeat still misses.
+    for (let attempt = 0; attempt < 10 && !FROM_CACHE.test(cache ?? ''); attempt++) {
+      if (attempt) await sleep(1000)
+      const response = await request(INTRO_PAGE.path)
+      expect(response.status).toBe(200)
+      await response.text()
+      cache = response.headers.get('x-vercel-cache')
+    }
+    expect(cache).toMatch(FROM_CACHE)
+  })
+})
+
+// The push webhook only targets production.
+describe.runIf(process.env.SMOKE_ENVIRONMENT === 'Production')('revalidation webhook', () => {
+  it('refuses an unsigned call', async () => {
+    const response = await request('/api/revalidate', { method: 'POST' })
+    // A 501 means `WEBHOOK_SECRET` or `VERCEL_BYPASS_TOKEN` is missing from the deployment.
+    expect(response.status).toBe(401)
+  })
+})
+
+// Every run costs AI Gateway calls, so it is opt-in.
+describe.runIf(process.env.SMOKE_ASSISTANT === 'true')('assistant', () => {
+  it('streams an answer', async () => {
+    const response = await request('/api/assistant', {
+      method: 'POST',
+      headers: { 'content-type': 'application/json' },
+      body: JSON.stringify({
+        messages: [{ id: 'smoke', role: 'user', parts: [{ type: 'text', text: 'In one sentence, what is this documentation about?' }] }],
+      }),
+      signal: AbortSignal.timeout(55_000),
+    })
+    expect(response.status).toBe(200)
+    expect(response.headers.get('content-type')).toContain('text/event-stream')
+
+    const stream = await response.text()
+    expect(stream).not.toContain('"type":"error"')
+    expect(stream).toContain('"type":"text-delta"')
+    expect(stream).toContain('"type":"finish"')
+  })
+})