Use fixed stores for CLI E2E tests

.github/workflows/tests-pr.yml

@@ -252,8 +252,14 @@ jobs:
     # Playwright — those failures belong to the build lane, not to E2E.
     needs: [type-check, bundle]
     runs-on: ubuntu-latest
-    timeout-minutes: 20
+    timeout-minutes: 45
     continue-on-error: true
+    env:
+      E2E_ACCOUNT_EMAIL: ${{ secrets.E2E_ACCOUNT_EMAIL }}
+      E2E_ACCOUNT_PASSWORD: ${{ secrets.E2E_ACCOUNT_PASSWORD }}
+      E2E_ORG_ID: ${{ secrets.E2E_ORG_ID }}
+      E2E_LOADTEST_HEADER: ${{ secrets.E2E_LOADTEST_HEADER }}
+      E2E_SHARD: ${{ matrix.shard }}
     strategy:
       fail-fast: false
       matrix:
@@ -276,13 +282,8 @@ jobs:
       - name: Rebuild node-pty
         run: pnpm rebuild node-pty
       - name: Run E2E tests
+        id: e2e
         working-directory: packages/e2e
-        env:
-          E2E_ACCOUNT_EMAIL: ${{ secrets.E2E_ACCOUNT_EMAIL }}
-          E2E_ACCOUNT_PASSWORD: ${{ secrets.E2E_ACCOUNT_PASSWORD }}
-          E2E_STORE_FQDN: ${{ secrets.E2E_STORE_FQDN }}
-          E2E_ORG_ID: ${{ secrets.E2E_ORG_ID }}
-          E2E_LOADTEST_HEADER: ${{ secrets.E2E_LOADTEST_HEADER }}
         run: pnpm exec playwright test --project remote --shard ${{ matrix.shard }}
       - name: Upload Playwright report
         uses: actions/upload-artifact@v4
@@ -304,58 +305,12 @@ jobs:
             !packages/e2e/test-results/**/trace.zip
           retention-days: 14
 
-  e2e-cleanup:
-    name: 'Cleanup current-run E2E resources'
-    needs: e2e-tests
-    if: ${{ always() && needs.e2e-tests.result != 'skipped' }}
-    runs-on: ubuntu-latest
-    timeout-minutes: 20
-    continue-on-error: true
-    steps:
-      - uses: actions/checkout@v6
-        with:
-          repository: ${{ github.repository }}
-          ref: ${{ github.sha }}
-          fetch-depth: 1
-      - name: Setup deps
-        uses: ./.github/actions/setup-cli-deps
-        with:
-          node-version: ${{ env.PLAYWRIGHT_NODE_VERSION }}
-      - name: Install Playwright Chromium
-        run: pnpm exec playwright install chromium
-        working-directory: packages/e2e
-      - name: Build CLI for cleanup auth
-        run: pnpm nx run cli:build
-      - name: Prime cleanup auth state
-        continue-on-error: true
-        env:
-          E2E_ACCOUNT_EMAIL: ${{ secrets.E2E_ACCOUNT_EMAIL }}
-          E2E_ACCOUNT_PASSWORD: ${{ secrets.E2E_ACCOUNT_PASSWORD }}
-          E2E_ORG_ID: ${{ secrets.E2E_ORG_ID }}
-          E2E_LOADTEST_HEADER: ${{ secrets.E2E_LOADTEST_HEADER }}
-        run: pnpm --filter e2e exec tsx scripts/prime-browser-auth.ts
       - name: Cleanup current-run E2E apps
-        if: ${{ always() }}
-        continue-on-error: true
-        env:
-          E2E_ACCOUNT_EMAIL: ${{ secrets.E2E_ACCOUNT_EMAIL }}
-          E2E_ACCOUNT_PASSWORD: ${{ secrets.E2E_ACCOUNT_PASSWORD }}
-          E2E_ORG_ID: ${{ secrets.E2E_ORG_ID }}
-          E2E_LOADTEST_HEADER: ${{ secrets.E2E_LOADTEST_HEADER }}
-        run: |
-          RUN_TOKEN=$(node -e "process.stdout.write(BigInt(process.env.GITHUB_RUN_ID).toString(36))")
-          pnpm --filter e2e exec tsx scripts/cleanup-apps.ts --pattern "r${RUN_TOKEN}"
-      - name: Cleanup current-run E2E stores
-        if: ${{ always() }}
-        continue-on-error: true
-        env:
-          E2E_ACCOUNT_EMAIL: ${{ secrets.E2E_ACCOUNT_EMAIL }}
-          E2E_ACCOUNT_PASSWORD: ${{ secrets.E2E_ACCOUNT_PASSWORD }}
-          E2E_ORG_ID: ${{ secrets.E2E_ORG_ID }}
-          E2E_LOADTEST_HEADER: ${{ secrets.E2E_LOADTEST_HEADER }}
+        if: ${{ always() && steps.e2e.outcome != 'skipped' }}
         run: |
           RUN_TOKEN=$(node -e "process.stdout.write(BigInt(process.env.GITHUB_RUN_ID).toString(36))")
-          pnpm --filter e2e exec tsx scripts/cleanup-stores.ts --pattern "r${RUN_TOKEN}"
+          SHARD_NUMBER=${E2E_SHARD%/*}
+          pnpm --filter e2e exec tsx scripts/cleanup-apps.ts --pattern "r${RUN_TOKEN}a${GITHUB_RUN_ATTEMPT}s${SHARD_NUMBER}-"
 
   type-diff:
     if: github.event.pull_request.head.repo.full_name == github.repository

docs/cli/testing-strategy.md

@@ -95,7 +95,7 @@ For Vitest problems, see [our troubleshooting page](troubleshooting.md) or the [
 
 End-to-end tests live under `packages/e2e` and are implemented using [Playwright](https://playwright.dev/). They test full user journeys by invoking the CLI and verifying outputs. Run them with `pnpm test:e2e`.
 
-See the [E2E setup guide](../../packages/e2e/README.md) for legacy resource cleanup.
+See the [E2E setup guide](../../packages/e2e/README.md) for persistent store configuration, CI isolation, and cleanup.
 
 ## GitHub Actions
 

packages/e2e/.env.example

@@ -10,6 +10,10 @@ E2E_ACCOUNT_PASSWORD=
 # CI secret: E2E_ORG_ID
 E2E_ORG_ID=
 
+# Optional: use this development store instead of each test's fixed assignment.
+# The test account must have access. Leave empty to use setup/constants.ts.
+E2E_STORE_FQDN=
+
 # Required: full X-Shopify-Loadtest-<UUID> header name
 # CI secret: E2E_LOADTEST_HEADER
 E2E_LOADTEST_HEADER=

packages/e2e/README.md

@@ -6,6 +6,56 @@ Run local tests without Shopify credentials:
 pnpm --filter e2e exec playwright test --project local
 ```
 
+## Persistent development stores
+
+App tests reuse existing stores. They never create or delete a store. Each test creates its own app and local files, then stops its dev process and attempts to remove its preview, uninstall its app, and delete the app. Stores can contain apps left by previous tests.
+
+Create dedicated development stores once in the E2E organization and give the E2E account access. Use the `no-delete-e2e-` name prefix: the store cleanup script refuses to delete these stores, including in `--force` mode.
+
+The store domains are defined in `E2E_STORES` in `setup/constants.ts`. Each of the seven dev cases selects a named store directly; store 8 is spare. Adding a test requires an explicit assignment, with no worker or shard allocation.
+
+Copy `.env.example` to `.env` and supply the account, organization, and load-test header. Set `E2E_STORE_FQDN` to override the fixed assignments for local development. The test account must have access to the selected store.
+
+Concurrent runs can use the same stores because each test creates a separate app and cleans only its own app and preview. Do not reuse the same app across concurrent runs. Keep local CLI processes in separate directories and use separate ports when running multiple suites on one machine.
+
+## Browser use
+
+Shared authentication and app deletion use the browser. Dev tests do not check store emptiness. App uninstall uses the Admin API. The legacy bulk cleanup scripts still use browser discovery, uninstall, and store-emptiness checks before store deletion.
+
+## CI setup
+
+Keep `E2E_ACCOUNT_EMAIL`, `E2E_ACCOUNT_PASSWORD`, `E2E_ORG_ID`, and `E2E_LOADTEST_HEADER` in Actions secrets. Store configuration does not require a repository variable.
+
+CI passes `--shard 1/2` or `--shard 2/2` to Playwright and runs four workers per shard. Store assignment does not depend on the shard or worker count. E2E jobs do not lock the stores across runs.
+
+App names and cleanup patterns include the run, attempt, and shard. Cleanup cannot select another run's apps or delete the fixed development stores.
+
+To select one shard locally:
+
+```sh
+pnpm --filter e2e exec playwright test --project remote --shard 1/2
+```
+
+## Store lifecycle coverage
+
+`tests/store-lifecycle.spec.ts` is the one exception to store reuse. It creates one temporary development store with `store create dev --json`, checks it with `store list --json`, and deletes it with `store delete --force --json`. It requires confirmed deletion and checks that the store is absent from the active store list. It runs once across the two remote shards and needs one free store slot in the E2E organization.
+
+The lifecycle commands do not use a browser. They use the CLI session from shared authentication. The temporary store has an `e2e-store-` name with the run marker and a random suffix. Cleanup runs even when the test fails or `E2E_SKIP_TEARDOWN=1`, with a separate timeout. It finds only the exact test name and refuses to delete protected stores. The test report includes the temporary name for recovery if the runner stops before cleanup or provisioning finishes late.
+
+Run only this test with:
+
+```sh
+pnpm --filter e2e exec playwright test tests/store-lifecycle.spec.ts --project remote --workers 1
+```
+
+## Failed or cancelled runs
+
+Remote app cleanup failures produce warnings with the app name, store, failed step, and recovery command. Each later cleanup step is still attempted. These warnings do not fail an otherwise successful test or block the store from subsequent tests. Failure to stop a CLI process remains an error because the process can hold ports and keep making requests.
+
+Recover leftover apps with `cleanup-apps.ts --pattern <run-attempt-shard>` or uninstall them in the store admin. Use the exact run marker from the failed run's app names and confirm that the run has stopped. Store reuse does not wait for this recovery.
+
+`E2E_SKIP_TEARDOWN=1` deliberately leaves test apps installed. Use the cleanup script to remove them after debugging; their presence does not block store reuse.
+
 ## Old store cleanup
 
 `cleanup-stores.ts` remains available for old temporary stores. It excludes the `no-delete-e2e-` prefix and all stores configured in `E2E_STORE_FQDNS` or `E2E_STORE_FQDN`. Full and delete modes use five browser workers and the existing `store delete` behavior. Force mode sends API deletion requests with twenty workers and skips app checks. Confirmed deletions and requests still awaiting confirmation are reported separately. Force deletion can leave app installation records that prevent app deletion until the backend clears them.

packages/e2e/playwright.config.ts

@@ -12,11 +12,11 @@ export default defineConfig({
   fullyParallel: true,
   forbidOnly: isCI,
   retries: 0,
-  workers: 10,
+  workers: 4,
   maxFailures: isCI ? 3 : 0, // Stop early in CI after 3 failures
   reporter: isCI ? [['html', {open: 'never'}], ['list']] : [['list']],
   timeout: TEST_TIMEOUT.default, // Heavy tests override via test.setTimeout()
-  globalTimeout: 20 * 60 * 1000,
+  globalTimeout: 30 * 60 * 1000,
 
   use: {
     trace: isCI ? 'on' : 'off',

packages/e2e/setup/admin-api.ts

@@ -25,7 +25,7 @@ interface AdminApiUninstallCtx {
  *      works here because the E2E org owns both the app and the dev store.
  *   3. Run the `appUninstall` mutation, which uninstalls the calling app.
  *
- * Throws on any failure so teardown surfaces the error instead of leaking.
+ * Throws on failure so teardown can report the app that needs cleanup.
  */
 export async function uninstallAppWithAdminApi(ctx: AdminApiUninstallCtx): Promise<void> {
   const clientId = extractClientId(ctx.appDir)
@@ -55,7 +55,10 @@ async function mintAppAccessTokenWithRetry(storeFqdn: string, clientId: string,
 }
 
 async function fetchClientSecret(ctx: AdminApiUninstallCtx): Promise<string> {
-  const result = await ctx.cli.exec(['app', 'env', 'show', '--path', ctx.appDir], {timeout: CLI_TIMEOUT.short})
+  const result = await ctx.cli.exec(['app', 'env', 'show', '--path', ctx.appDir], {
+    timeout: CLI_TIMEOUT.short,
+    quiet: true,
+  })
   if (result.exitCode !== 0) {
     throw new Error(`app env show failed (exit ${result.exitCode}): ${result.stderr}`)
   }
@@ -69,6 +72,7 @@ async function fetchClientSecret(ctx: AdminApiUninstallCtx): Promise<string> {
 async function mintAppAccessToken(storeFqdn: string, clientId: string, clientSecret: string): Promise<string> {
   const response = await fetch(`https://${storeFqdn}/admin/oauth/access_token`, {
     method: 'POST',
+    signal: AbortSignal.timeout(CLI_TIMEOUT.short),
     headers: {'Content-Type': 'application/x-www-form-urlencoded', ...loadtestHeaderRecord()},
     body: new URLSearchParams({
       grant_type: 'client_credentials',
@@ -92,6 +96,7 @@ async function mintAppAccessToken(storeFqdn: string, clientId: string, clientSec
 async function runAppUninstall(storeFqdn: string, accessToken: string): Promise<void> {
   const response = await fetch(`https://${storeFqdn}/admin/api/${ADMIN_API_VERSION}/graphql.json`, {
     method: 'POST',
+    signal: AbortSignal.timeout(CLI_TIMEOUT.short),
     headers: {
       'Content-Type': 'application/json',
       'X-Shopify-Access-Token': accessToken,
@@ -111,7 +116,10 @@ async function runAppUninstall(storeFqdn: string, accessToken: string): Promise<
   if (payload.errors) {
     throw new Error(`appUninstall returned errors: ${JSON.stringify(payload.errors)}`)
   }
-  const userErrors = payload.data?.appUninstall?.userErrors ?? []
+  const userErrors = payload.data?.appUninstall?.userErrors
+  if (!Array.isArray(userErrors)) {
+    throw new Error('appUninstall returned an invalid response; uninstall was not confirmed')
+  }
   if (userErrors.length > 0) {
     throw new Error(`appUninstall returned user errors: ${userErrors.map((error) => error.message).join(', ')}`)
   }

packages/e2e/setup/app.ts

@@ -111,10 +111,22 @@ export function extractClientId(appDir: string, configName?: string): string {
 }
 
 /**
- * Build the direct Dev Dashboard URL for an app from its local TOML client_id.
+ * Resolve the numeric app ID used by Dev Dashboard routes, which differ from client IDs.
  */
-export function devDashboardAppUrl(appDir: string, orgId: string, configName?: string): string {
-  return `https://dev.shopify.com/dashboard/${orgId}/apps/${extractClientId(appDir, configName)}`
+export async function devDashboardAppUrl(ctx: {
+  cli: Pick<CLIProcess, 'exec'>
+  appDir: string
+  orgId: string
+  configName?: string
+}): Promise<string> {
+  const args = ['app', 'info', '--json', '--path', ctx.appDir]
+  if (ctx.configName) args.push('--config', ctx.configName)
+  const result = await ctx.cli.exec(args, {quiet: true})
+  if (result.exitCode !== 0) throw new Error(`Can't resolve the app's dashboard URL: ${result.stderr}`)
+  const info = JSON.parse(result.stdout) as {remoteApp?: {id?: string}}
+  const appId = info.remoteApp?.id?.match(/^(?:gid:\/\/shopify\/App\/)?(\d+)$/)?.[1]
+  if (!appId) throw new Error("app info didn't return a valid numeric app ID")
+  return `https://dev.shopify.com/dashboard/${ctx.orgId}/apps/${appId}`
 }
 
 /**
@@ -328,7 +340,7 @@ export async function findAppOnDevDashboard(page: Page, appName: string, orgId?:
   const org = orgId ?? (process.env.E2E_ORG_ID ?? '').trim()
   const email = process.env.E2E_ACCOUNT_EMAIL
 
-  await navigateToDashboard({browserPage: page, email, orgId: org})
+  await navigateToDashboard({browserPage: page, email, orgId: org, searchTerm: appName})
 
   // Scan current page + pagination for the app
   while (true) {

packages/e2e/setup/cli.ts

@@ -40,11 +40,16 @@ export interface SpawnedProcess {
 
 export interface CLIProcess {
   /** Execute a CLI command non-interactively via execa */
-  exec(args: string[], opts?: {cwd?: string; env?: NodeJS.ProcessEnv; timeout?: number}): Promise<ExecResult>
+  exec(
+    args: string[],
+    opts?: {cwd?: string; env?: NodeJS.ProcessEnv; timeout?: number; quiet?: boolean},
+  ): Promise<ExecResult>
   /** Execute the create-app binary non-interactively via execa */
   execCreateApp(args: string[], opts?: {cwd?: string; env?: NodeJS.ProcessEnv; timeout?: number}): Promise<ExecResult>
   /** Spawn an interactive CLI command via node-pty */
   spawn(args: string[], opts?: {cwd?: string; env?: NodeJS.ProcessEnv}): Promise<SpawnedProcess>
+  /** Stop all owned interactive processes before resetting their remote state. */
+  stopAll(): Promise<void>
 }
 
 /**
@@ -59,16 +64,20 @@ export const cliFixture = envFixture.extend<{cli: CLIProcess}>({
     // When DEBUG=1, tee the subprocess streams to the parent so the CLI's
     // info/success boxes and progress messages appear live, while still
     // letting execa buffer and return the captured output.
-    const runExeca = (bin: string, args: string[], execaOpts: ExecaOptions) => {
+    const runExeca = (bin: string, args: string[], execaOpts: ExecaOptions, quiet = false) => {
       const subprocess = execa('node', [bin, ...args], execaOpts)
-      if (process.env.DEBUG === '1') {
+      if (process.env.DEBUG === '1' && !quiet) {
         subprocess.stdout?.pipe(process.stdout, {end: false})
         subprocess.stderr?.pipe(process.stderr, {end: false})
       }
       return subprocess
     }
 
     const cli: CLIProcess = {
+      async stopAll() {
+        for (const proc of spawnedProcesses) proc.kill()
+        await Promise.all(spawnedProcesses.map((proc) => proc.waitForExit(CLI_TIMEOUT.short)))
+      },
       async exec(args, opts = {}) {
         const timeout = opts.timeout ?? CLI_TIMEOUT.medium
         const execaOpts: ExecaOptions = {
@@ -81,7 +90,7 @@ export const cliFixture = envFixture.extend<{cli: CLIProcess}>({
         cliLog.log(env, `exec: node ${executables.cli}`)
         cliLog.log(env, args.join(' '))
 
-        const result = await runExeca(executables.cli, args, execaOpts)
+        const result = await runExeca(executables.cli, args, execaOpts, opts.quiet)
 
         return {
           stdout: result.stdout ?? '',
@@ -154,17 +163,20 @@ export const cliFixture = envFixture.extend<{cli: CLIProcess}>({
         })
 
         let exitCode: number | undefined
-        let exitResolve: ((code: number) => void) | undefined
+        const exitListeners = new Set<(code: number) => void>()
 
         ptyProcess.onExit(({exitCode: code}) => {
           exitCode = code
-          if (exitResolve) {
-            exitResolve(code)
-          }
+          for (const listener of exitListeners) listener(code)
+          exitListeners.clear()
           // Reject any remaining output waiters. reject() removes each waiter
           // from outputWaiters, so iterate over a snapshot to avoid skipping.
           for (const waiter of [...outputWaiters]) {
-            waiter.reject(new Error(`Process exited (code ${code}) while waiting for output: "${waiter.text}"`))
+            waiter.reject(
+              new Error(
+                `Process exited (code ${code}) while waiting for output: "${waiter.text}"\n\nCaptured output:\n${stripAnsi(output)}`,
+              ),
+            )
           }
         })
 
@@ -247,17 +259,20 @@ export const cliFixture = envFixture.extend<{cli: CLIProcess}>({
 
             return new Promise<number>((resolve, reject) => {
               const timer = setTimeout(() => {
+                exitListeners.delete(onExit)
                 reject(new Error(`Timed out after ${timeoutMs}ms waiting for process exit`))
               }, timeoutMs)
 
-              exitResolve = (code) => {
+              const onExit = (code: number) => {
                 clearTimeout(timer)
                 resolve(code)
               }
+              exitListeners.add(onExit)
             })
           },
 
           kill() {
+            if (exitCode !== undefined) return
             try {
               ptyProcess.kill()
               // eslint-disable-next-line no-catch-all/no-catch-all
@@ -276,11 +291,10 @@ export const cliFixture = envFixture.extend<{cli: CLIProcess}>({
       },
     }
 
-    await use(cli)
-
-    // Teardown: kill all spawned processes
-    for (const proc of spawnedProcesses) {
-      proc.kill()
+    try {
+      await use(cli)
+    } finally {
+      await cli.stopAll()
     }
   },
 })

packages/e2e/setup/constants.ts

@@ -1,3 +1,14 @@
+export const E2E_STORES = {
+  devServer: 'no-delete-e2e-reusable-store-1.myshopify.com',
+  configReload: 'no-delete-e2e-reusable-store-2.myshopify.com',
+  extensionCreation: 'no-delete-e2e-reusable-store-3.myshopify.com',
+  extensionDeletion: 'no-delete-e2e-reusable-store-4.myshopify.com',
+  namedConfig: 'no-delete-e2e-reusable-store-5.myshopify.com',
+  defaultConfig: 'no-delete-e2e-reusable-store-6.myshopify.com',
+  fullConfig: 'no-delete-e2e-reusable-store-7.myshopify.com',
+  spare: 'no-delete-e2e-reusable-store-8.myshopify.com',
+} as const
+
 // ---------------------------------------------------------------------------
 // Shared timeout constants for E2E tests
 // ---------------------------------------------------------------------------