From 9e3c59688f4c714c20236fce528dfa76ff2253a8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Isaac=20Rold=C3=A1n?= Date: Mon, 19 Oct 2026 17:38:45 +0200 Subject: [PATCH 01/14] Add reusable dev session files to CLI Kit --- .changeset/cli-kit-dev-instance.md | 5 + packages/cli-kit/package.json | 2 + .../src/public/node/dev-instance.test.ts | 402 ++++++++++++++ .../cli-kit/src/public/node/dev-instance.ts | 506 ++++++++++++++++++ pnpm-lock.yaml | 6 + 5 files changed, 921 insertions(+) create mode 100644 .changeset/cli-kit-dev-instance.md create mode 100644 packages/cli-kit/src/public/node/dev-instance.test.ts create mode 100644 packages/cli-kit/src/public/node/dev-instance.ts diff --git a/.changeset/cli-kit-dev-instance.md b/.changeset/cli-kit-dev-instance.md new file mode 100644 index 00000000000..6a048e49412 --- /dev/null +++ b/.changeset/cli-kit-dev-instance.md @@ -0,0 +1,5 @@ +--- +'@shopify/cli-kit': minor +--- + +Add dev session files for ownership, status, and JSONL logs. diff --git a/packages/cli-kit/package.json b/packages/cli-kit/package.json index c7cc524f7c5..37ae101c481 100644 --- a/packages/cli-kit/package.json +++ b/packages/cli-kit/package.json @@ -147,6 +147,7 @@ "node-fetch": "3.3.2", "open": "8.4.2", "pathe": "1.1.2", + "proper-lockfile": "4.1.2", "react": "19.2.4", "semver": "7.8.4", "react-dom": "19.2.4", @@ -163,6 +164,7 @@ "@types/fs-extra": "9.0.13", "@types/gradient-string": "^1.1.2", "@types/lodash": "4.17.24", + "@types/proper-lockfile": "4.1.4", "@types/react": "^19.0.0", "@types/react-dom": "^19.0.0", "@types/semver": "^7.5.2", diff --git a/packages/cli-kit/src/public/node/dev-instance.test.ts b/packages/cli-kit/src/public/node/dev-instance.test.ts new file mode 100644 index 00000000000..8a0d8c647fb --- /dev/null +++ b/packages/cli-kit/src/public/node/dev-instance.test.ts @@ -0,0 +1,402 @@ +import { + acquireDevInstance, + DEV_INSTANCE_ALREADY_RUNNING_CODE, + devInstancePaths, + isProcessAlive, + listDevInstances, + readDevInstance, +} from './dev-instance.js' +import {AbortError} from './error.js' +import {fileExists, inTemporaryDirectory, mkdir, readFile, rmdir, writeFile} from './fs.js' +import {describe, expect, test, vi} from 'vitest' +import {Writable} from 'stream' +import {utimes} from 'fs/promises' + +const DEAD_PID = 2 ** 22 + 12345 + +function collectingStream() { + const chunks: string[] = [] + const stream = new Writable({ + write(chunk, _encoding, callback) { + chunks.push(chunk.toString()) + callback() + }, + }) + return {stream, chunks} +} + +async function readLog(path: string) { + const content = await readFile(path) + return content + .split('\n') + .filter(Boolean) + .map((line) => JSON.parse(line)) +} + +describe('acquireDevInstance', () => { + test('describes the instance in .shopify/dev and releases it', async () => { + await inTemporaryDirectory(async (directory) => { + // When + const instance = await acquireDevInstance({ + command: 'app dev', + directory, + key: 'shopify.app.toml', + details: {store: 'my-store.myshopify.com'}, + }) + + // Then + const paths = devInstancePaths({command: 'app dev', directory, key: 'shopify.app.toml'}) + expect(paths.infoFile).toMatch(/\.shopify\/dev\/app-dev-shopify\.app\.toml\.json$/) + const info = JSON.parse(await readFile(paths.infoFile)) + expect(info).toMatchObject({ + schemaVersion: 1, + command: 'app dev', + key: 'shopify.app.toml', + pid: process.pid, + status: 'starting', + directory, + logFile: paths.logFile, + details: {store: 'my-store.myshopify.com'}, + }) + await expect(fileExists(paths.lockFile)).resolves.toBe(true) + await expect(readDevInstance({command: 'app dev', directory, key: 'shopify.app.toml'})).resolves.toMatchObject({ + pid: process.pid, + }) + + // When + await instance.release() + await instance.release() + + // Then + await expect(fileExists(paths.infoFile)).resolves.toBe(false) + await expect(fileExists(paths.lockFile)).resolves.toBe(false) + await expect(readDevInstance({command: 'app dev', directory, key: 'shopify.app.toml'})).resolves.toBeUndefined() + }) + }) + + test('refuses to start while another live instance holds the lock', async () => { + await inTemporaryDirectory(async (directory) => { + // Given + const running = await acquireDevInstance({command: 'app dev', directory, details: {previewUrl: 'https://x'}}) + + // When + const error = await acquireDevInstance({command: 'app dev', directory}).catch((error: unknown) => error) + + // Then + expect(error).toBeInstanceOf(AbortError) + expect((error as AbortError).message).toContain('Another `shopify app dev` is already running') + expect((error as AbortError).code).toBe(DEV_INSTANCE_ALREADY_RUNNING_CODE) + expect((error as AbortError).details).toEqual({instance: running.info}) + await running.release() + }) + }) + + test('takes over a lock left behind by a process that no longer exists', async () => { + await inTemporaryDirectory(async (directory) => { + // Given + const paths = devInstancePaths({command: 'theme dev', directory}) + await mkdir(paths.lockFile) + await writeFile(paths.infoFile, JSON.stringify({pid: DEAD_PID, logFile: paths.logFile, details: {}})) + + // When + const instance = await acquireDevInstance({command: 'theme dev', directory}) + + // Then + expect(instance.info.pid).toBe(process.pid) + await instance.release() + }) + }) + + test('does not take over a live owner with a stale heartbeat', async () => { + await inTemporaryDirectory(async (directory) => { + const paths = devInstancePaths({command: 'app dev', directory}) + await mkdir(paths.lockFile) + const oneMinuteAgo = new Date(Date.now() - 60_000) + await utimes(paths.lockFile, oneMinuteAgo, oneMinuteAgo) + await writeFile(paths.infoFile, JSON.stringify({pid: process.ppid, logFile: paths.logFile, details: {}})) + + // When + await expect(acquireDevInstance({command: 'app dev', directory})).rejects.toMatchObject({ + code: DEV_INSTANCE_ALREADY_RUNNING_CODE, + }) + }) + }) + + test('only one contender recovers an abandoned instance', async () => { + await inTemporaryDirectory(async (directory) => { + const paths = devInstancePaths({command: 'app dev', directory}) + await mkdir(paths.lockFile) + await writeFile(paths.infoFile, JSON.stringify({pid: DEAD_PID, details: {}})) + const attempts = await Promise.allSettled( + Array.from({length: 8}, async () => acquireDevInstance({command: 'app dev', directory})), + ) + const owners = attempts.flatMap((attempt) => (attempt.status === 'fulfilled' ? [attempt.value] : [])) + try { + expect(owners).toHaveLength(1) + await expect(readDevInstance({command: 'app dev', directory})).resolves.toMatchObject({ + sessionId: owners[0]?.info.sessionId, + }) + for (const attempt of attempts) { + if (attempt.status === 'rejected') expect(attempt.reason.code).toBe(DEV_INSTANCE_ALREADY_RUNNING_CODE) + } + } finally { + await Promise.all(owners.map((owner) => owner.release())) + } + }) + }) + + test('requires explicit cleanup of an interrupted startup', async () => { + await inTemporaryDirectory(async (directory) => { + const identity = {command: 'app dev', directory} + const paths = devInstancePaths(identity) + const startupLock = `${paths.lockFile}.acquire` + await mkdir(startupLock) + await expect(acquireDevInstance(identity)).rejects.toMatchObject({ + code: DEV_INSTANCE_ALREADY_RUNNING_CODE, + message: expect.stringContaining('a previous startup was interrupted'), + }) + await expect(fileExists(startupLock)).resolves.toBe(true) + await expect(fileExists(paths.lockFile)).resolves.toBe(false) + await rmdir(startupLock) + const instance = await acquireDevInstance(identity) + await instance.release() + await expect(fileExists(startupLock)).resolves.toBe(false) + }) + }) + + test('does not remove another session record or lock during release', async () => { + await inTemporaryDirectory(async (directory) => { + const instance = await acquireDevInstance({command: 'app dev', directory}) + const replacement = {...instance.info, sessionId: 'replacement'} + await writeFile(instance.paths.infoFile, JSON.stringify(replacement)) + await instance.release() + expect(JSON.parse(await readFile(instance.paths.infoFile))).toEqual(replacement) + await expect(fileExists(instance.paths.lockFile)).resolves.toBe(true) + }) + }) + + test('refuses to start while the lock is fresh but not described yet', async () => { + await inTemporaryDirectory(async (directory) => { + // Given + const paths = devInstancePaths({command: 'app dev', directory}) + await mkdir(paths.lockFile) + + // When + const error = await acquireDevInstance({command: 'app dev', directory}).catch((error: unknown) => error) + + // Then + expect((error as AbortError).message).toContain('is starting for this project') + expect((error as AbortError).code).toBe(DEV_INSTANCE_ALREADY_RUNNING_CODE) + }) + }) + + test('aborts when its lock is removed', async () => { + await inTemporaryDirectory(async (directory) => { + vi.useFakeTimers({toFake: ['setTimeout', 'clearTimeout']}) + const instance = await acquireDevInstance({command: 'app dev', directory}) + try { + await rmdir(instance.paths.lockFile) + await vi.advanceTimersByTimeAsync(15_000) + await vi.waitFor(() => expect(instance.abortSignal.aborted).toBe(true)) + expect(instance.abortSignal.reason).toBeInstanceOf(AbortError) + } finally { + await instance.release() + vi.useRealTimers() + } + }) + }) + + test('keeps instances with different keys apart', async () => { + await inTemporaryDirectory(async (directory) => { + // When + const production = await acquireDevInstance({command: 'app dev', directory, key: 'shopify.app.toml'}) + const staging = await acquireDevInstance({command: 'app dev', directory, key: 'shopify.app.staging.toml'}) + + // Then + const instances = await listDevInstances(directory) + expect(instances.map((instance) => instance.key).sort()).toEqual(['shopify.app.staging.toml', 'shopify.app.toml']) + await production.release() + await staging.release() + await expect(listDevInstances(directory)).resolves.toEqual([]) + }) + }) + + test('persists updates to the instance description', async () => { + await inTemporaryDirectory(async (directory) => { + // Given + const instance = await acquireDevInstance({command: 'app dev', directory, details: {store: 'a'}}) + + // When + await Promise.all([ + instance.update({status: 'running'}), + instance.update({details: {previewUrl: 'https://preview'}}), + ]) + + // Then + const info = await readDevInstance({command: 'app dev', directory}) + expect(info).toMatchObject({status: 'running', details: {store: 'a', previewUrl: 'https://preview'}}) + await instance.release() + }) + }) + + test('can save status again after a write fails', async () => { + await inTemporaryDirectory(async (directory) => { + const instance = await acquireDevInstance({command: 'app dev', directory}) + const temporaryFile = `${instance.paths.infoFile}.${instance.info.sessionId}.tmp` + try { + await mkdir(temporaryFile) + await expect(instance.update({details: {sequence: 1}})).rejects.toThrow() + await rmdir(temporaryFile) + await instance.update({details: {sequence: 2}}) + expect(JSON.parse(await readFile(instance.paths.infoFile))).toHaveProperty('details.sequence', 2) + } finally { + await rmdir(temporaryFile, {force: true}) + await instance.release() + } + }) + }) + + test.each(['infoFile', 'logFile'] as const)('releases startup resources when %s cannot be written', async (file) => { + await inTemporaryDirectory(async (directory) => { + const paths = devInstancePaths({command: 'app dev', directory}) + await mkdir(paths[file]) + const exitListeners = process.listenerCount('exit') + await expect(acquireDevInstance({command: 'app dev', directory})).rejects.toThrow() + await expect(fileExists(paths.lockFile)).resolves.toBe(false) + await expect(fileExists(`${paths.lockFile}.acquire`)).resolves.toBe(false) + expect(process.listenerCount('exit')).toBe(exitListeners) + await rmdir(paths[file]) + const restarted = await acquireDevInstance({command: 'app dev', directory}) + await restarted.log({source: 'web', message: 'recovered'}) + await restarted.release() + await expect(readLog(paths.logFile)).resolves.toEqual([expect.objectContaining({message: 'recovered'})]) + }) + }) +}) + +describe('DevInstance log', () => { + test('tees process output into a JSON Lines log without ANSI styling', async () => { + await inTemporaryDirectory(async (directory) => { + // Given + const instance = await acquireDevInstance({command: 'app dev', directory}) + const {stream: target, chunks} = collectingStream() + const stdout = instance.tee(target, {source: 'web', stream: 'stdout'}) + + // When + stdout.write('\u001b[32mready\u001b[39m in 20ms\npartial ') + stdout.write('line\n\n') + await new Promise((resolve) => stdout.end(resolve)) + await instance.log({source: 'app dev', level: 'info', message: 'Preview URL: https://preview'}) + await instance.release() + + // Then + expect(chunks.join('')).toBe('\u001b[32mready\u001b[39m in 20ms\npartial line\n\n') + const entries = await readLog(instance.paths.logFile) + expect(entries.map(({sessionId, ...entry}) => entry)).toEqual([ + {timestamp: expect.any(String), source: 'web', stream: 'stdout', message: 'ready in 20ms'}, + {timestamp: expect.any(String), source: 'web', stream: 'stdout', message: 'partial line'}, + {timestamp: expect.any(String), source: 'app dev', level: 'info', message: 'Preview URL: https://preview'}, + ]) + expect(entries.every((entry) => entry.sessionId === instance.info.sessionId)).toBe(true) + expect(entries[0].timestamp).toMatch(/^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}Z$/) + }) + }) + + test('flushes a split UTF-8 line on release without requiring the source to end', async () => { + await inTemporaryDirectory(async (directory) => { + const instance = await acquireDevInstance({command: 'app dev', directory}) + const output = instance.tee(collectingStream().stream, {source: 'web', stream: 'stdout'}) + const bytes = Buffer.from('café') + output.write(bytes.subarray(0, 4)) + output.write(bytes.subarray(4)) + await instance.release() + await expect(readLog(instance.paths.logFile)).resolves.toEqual([ + expect.objectContaining({sessionId: instance.info.sessionId, source: 'web', message: 'café'}), + ]) + }) + }) + + test('starts each run with an empty log', async () => { + await inTemporaryDirectory(async (directory) => { + // Given + const first = await acquireDevInstance({command: 'app dev', directory}) + await first.log({source: 'app dev', message: 'first run'}) + await first.release() + + // When + const second = await acquireDevInstance({command: 'app dev', directory}) + expect(second.info.sessionId).not.toBe(first.info.sessionId) + await second.log({source: 'app dev', message: 'second run'}) + await second.release() + + // Then + const entries = await readLog(second.paths.logFile) + expect(entries.map((entry) => entry.message)).toEqual(['second run']) + }) + }) + + test('holds back the producer until the destination accepts its output', async () => { + await inTemporaryDirectory(async (directory) => { + const instance = await acquireDevInstance({command: 'app dev', directory}) + let accept: () => void = () => {} + const target = new Writable({ + write(_chunk, _encoding, callback) { + accept = () => { + accept = () => {} + callback() + } + }, + }) + const output = instance.tee(target, {source: 'web', stream: 'stdout'}) + let written = false + const completed = new Promise((resolve, reject) => { + const available = output.write(`${'x'.repeat(output.writableHighWaterMark)}\n`, (error) => { + if (error) return reject(error) + written = true + resolve() + }) + expect(available).toBe(false) + }) + try { + expect(written).toBe(false) + accept() + await completed + await instance.release() + await expect(readLog(instance.paths.logFile)).resolves.toHaveLength(1) + expect(target.listenerCount('error')).toBe(0) + } finally { + accept() + await instance.release() + } + }) + }) + + test('reports destination errors and releases the instance after output fails', async () => { + await inTemporaryDirectory(async (directory) => { + const instance = await acquireDevInstance({command: 'app dev', directory}) + const target = new Writable({ + write(_chunk, _encoding, callback) { + callback(new Error('Output failed')) + }, + }) + const output = instance.tee(target, {source: 'web', stream: 'stdout'}) + await expect( + new Promise((resolve, reject) => { + output.write('message\n', (error) => (error ? reject(error) : resolve())) + }), + ).rejects.toThrow('Output failed') + await expect(instance.release()).rejects.toThrow('Output failed') + expect(instance.abortSignal.aborted).toBe(true) + await expect(fileExists(instance.paths.infoFile)).resolves.toBe(false) + await expect(fileExists(instance.paths.lockFile)).resolves.toBe(false) + }) + }) +}) + +describe('isProcessAlive', () => { + test('detects the current process and a missing one', () => { + expect(isProcessAlive(process.pid)).toBe(true) + expect(isProcessAlive(DEAD_PID)).toBe(false) + expect(isProcessAlive(0)).toBe(false) + }) +}) diff --git a/packages/cli-kit/src/public/node/dev-instance.ts b/packages/cli-kit/src/public/node/dev-instance.ts new file mode 100644 index 00000000000..346322a4f06 --- /dev/null +++ b/packages/cli-kit/src/public/node/dev-instance.ts @@ -0,0 +1,506 @@ +import {getOrCreateHiddenShopifyFolder} from './hidden-folder.js' +import {joinPath} from './path.js' +import {fileExists, glob, mkdir, readFile, removeFile, removeFileSync, renameFile, rmdir} from './fs.js' +import {AbortError} from './error.js' +import {outputDebug, unstyled} from './output.js' +import {formatJsonOutputTimestamp} from './json-output-schema.js' +import {randomUUID} from './crypto.js' +import lockfile from 'proper-lockfile' +import fs, {createWriteStream, readFileSync, type WriteStream} from 'fs' +import {mkdir as mkdirExclusive, rmdir as removeEmptyDirectory, writeFile as writeFileWithMode} from 'fs/promises' +import {Writable} from 'stream' +import {finished} from 'stream/promises' +import {StringDecoder} from 'string_decoder' +import {once} from 'events' + +/** + * A stable error code for JSON consumers when a long-running command refuses to start because another instance of + * it is already running for the same project. + */ +export const DEV_INSTANCE_ALREADY_RUNNING_CODE = 'dev_instance_already_running' + +const DEV_INSTANCES_FOLDER = 'dev' +const DEFAULT_STALE_MS = 30_000 + +/** + * Describes a running instance of a long-running command such as `app dev` or `theme dev`. + * It's written to `.shopify/dev/.json` so agents and other tools can discover the instance. + */ +export interface DevInstanceInfo { + schemaVersion: 1 + /** Identifies this run, including when the OS reuses a PID. */ + sessionId: string + /** The command that owns the instance, for example `app dev`. */ + command: string + /** Distinguishes instances of the same command in the same directory, for example the app configuration file. */ + key?: string + pid: number + status: 'starting' | 'running' + startedAt: string + updatedAt: string + /** The project directory the instance runs for. */ + directory: string + /** The JSON Lines file the instance appends its output to. */ + logFile: string + /** Command-specific, JSON-serializable details such as URLs and ports. */ + details: {[key: string]: unknown} +} + +/** One line of a dev instance's JSON Lines log. */ +export interface DevInstanceLogEntry { + sessionId: string + timestamp: string + /** What produced the line, for example a process prefix like `web` or `extensions`. */ + source: string + /** The stream the line was written to, when it comes from a process. */ + stream?: 'stdout' | 'stderr' + level?: 'debug' | 'info' | 'warning' | 'error' + message: string +} + +export interface DevInstancePaths { + /** The folder that holds every dev instance of the project: `.shopify/dev`. */ + folder: string + /** The instance description. */ + infoFile: string + /** The lock directory, owned by `proper-lockfile`. */ + lockFile: string + /** The JSON Lines log. */ + logFile: string +} + +export interface DevInstanceIdentity { + /** The command that owns the instance, for example `app dev`. */ + command: string + /** The project directory. */ + directory: string + /** Distinguishes several instances of the same command in the same directory. */ + key?: string +} + +export interface AcquireDevInstanceOptions extends DevInstanceIdentity { + /** Initial command-specific details. */ + details?: {[key: string]: unknown} +} + +export interface DevInstanceUpdate { + status?: DevInstanceInfo['status'] + /** Merged into the existing details. */ + details?: {[key: string]: unknown} +} + +export interface DevInstance { + readonly info: DevInstanceInfo + readonly paths: DevInstancePaths + /** Aborts when this session loses its lock or output fails. Consumers must stop their owned work. */ + readonly abortSignal: AbortSignal + /** Updates and persists the instance description. */ + update: (changes: DevInstanceUpdate) => Promise + /** Appends one entry to the JSON Lines log. */ + log: (entry: Omit) => Promise + /** + * Wraps a stream so every line written to it is also appended to the log, without ANSI styling. + * The returned stream forwards the original bytes to `target` unchanged. + */ + tee: (target: Writable, options: {source: string; stream: 'stdout' | 'stderr'}) => Writable + /** Closes the log, removes the instance description and releases the lock. Safe to call more than once. */ + release: () => Promise +} + +/** + * Returns the files used by a dev instance. Every instance of a project shares `.shopify/dev`, + * with one `.json`, `.lock` and `.jsonl` per command and key. + * + * @param identity - The command, directory and optional key of the instance. + * @returns The paths of the instance files. + */ +export function devInstancePaths(identity: DevInstanceIdentity): DevInstancePaths { + const folder = joinPath(identity.directory, '.shopify', DEV_INSTANCES_FOLDER) + const slug = devInstanceSlug(identity) + return { + folder, + infoFile: joinPath(folder, `${slug}.json`), + lockFile: joinPath(folder, `${slug}.lock`), + logFile: joinPath(folder, `${slug}.jsonl`), + } +} + +/** + * Marks this process as the running instance of a long-running command for a project. + * + * Mutual exclusion uses `proper-lockfile`: an atomic `mkdir` lock whose mtime this process refreshes as a heartbeat. + * An exclusive startup directory serializes abandoned-lock recovery. An interrupted startup requires explicit + * cleanup of that directory. A live owner blocks takeover, including when its heartbeat is stale. + * + * @param options - The instance identity, initial details. + * @returns The acquired instance. + * @throws AbortError with code {@link DEV_INSTANCE_ALREADY_RUNNING_CODE} when another live instance holds the lock. + */ +export async function acquireDevInstance(options: AcquireDevInstanceOptions): Promise { + const paths = devInstancePaths(options) + await getOrCreateHiddenShopifyFolder(options.directory) + await mkdir(paths.folder) + const startupLock = `${paths.lockFile}.acquire` + try { + await mkdirExclusive(startupLock, {mode: 0o700}) + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== 'EEXIST') throw error + const startupError = new AbortError( + `Another \`shopify ${options.command}\` is starting, or a previous startup was interrupted.`, + 'Wait for other starts to finish. If none is running, remove the startup lock below and try again.', + [['Startup lock', {filePath: startupLock}]], + ) + startupError.code = DEV_INSTANCE_ALREADY_RUNNING_CODE + throw startupError + } + try { + return await createDevInstance(options, paths) + } catch (error) { + if (!isLockedError(error)) throw error + const existing = await readDevInstanceInfo(paths.infoFile) + throw devInstanceAlreadyRunningError(options, existing && isProcessAlive(existing.pid) ? existing : undefined) + } finally { + await removeEmptyDirectory(startupLock) + } +} + +async function createDevInstance(options: AcquireDevInstanceOptions, paths: DevInstancePaths): Promise { + const previous = await readDevInstanceInfo(paths.infoFile) + if (previous && isProcessAlive(previous.pid)) throw devInstanceAlreadyRunningError(options, previous) + const sessionId = randomUUID() + const abortController = new AbortController() + let compromised = false + const ownsInfo = () => { + if (compromised) return false + try { + return (JSON.parse(readFileSync(paths.infoFile, 'utf8')) as DevInstanceInfo).sessionId === sessionId + // A missing or unreadable record does not authorize deleting another session's files. + // eslint-disable-next-line no-catch-all/no-catch-all + } catch { + return false + } + } + let published = false + const removeOwnedLockSync = (path: fs.PathLike) => { + if (published && !ownsInfo()) return + try { + fs.rmdirSync(path) + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== 'ENOENT') throw error + } + } + const lockOptions = { + realpath: false, + lockfilePath: paths.lockFile, + stale: DEFAULT_STALE_MS, + retries: 0, + // The default handler throws asynchronously and would crash the long-running command. + onCompromised: (error: Error) => { + compromised = true + outputDebug(`The ${options.command} instance lock was compromised: ${error.message}`) + abortController.abort( + new AbortError(`The ${options.command} instance lost its lock.`, 'Start the command again.'), + ) + }, + fs: { + ...fs, + rmdir: (path: fs.PathLike, callback: fs.NoParamCallback) => { + if (published && !ownsInfo()) return callback(null) + // proper-lockfile requires the callback filesystem API. + // eslint-disable-next-line n/prefer-promises/fs + fs.rmdir(path, callback) + }, + rmdirSync: removeOwnedLockSync, + }, + } + + if (previous) await rmdir(paths.lockFile, {force: true}) + const releaseLock = await lockfile.lock(paths.infoFile, lockOptions) + + const now = formatJsonOutputTimestamp(new Date()) + const info: DevInstanceInfo = { + schemaVersion: 1, + sessionId, + command: options.command, + ...(options.key ? {key: options.key} : {}), + pid: process.pid, + status: 'starting', + startedAt: now, + updatedAt: now, + directory: options.directory, + logFile: paths.logFile, + details: {...(options.details ?? {})}, + } + + // Each run starts a fresh log. + const logStream: WriteStream = createWriteStream(paths.logFile, {flags: 'w', mode: 0o600}) + logStream.on('error', (error) => abortController.abort(error)) + + let updates = Promise.resolve() + const outputs = new Set() + let released = false + let releasePromise: Promise | undefined + const removeInfoOnExit = () => { + // Remove the lock before the live owner's record, so a new starter cannot enter during cleanup. + if (!released && ownsInfo()) { + removeOwnedLockSync(paths.lockFile) + removeFileSync(paths.infoFile) + } + } + process.once('exit', removeInfoOnExit) + + const writeLogEntry = async (entry: Omit) => { + if (released || compromised) return + const line: DevInstanceLogEntry = { + sessionId: info.sessionId, + timestamp: formatJsonOutputTimestamp(new Date()), + ...entry, + } + await writeToStream(logStream, `${JSON.stringify(line)}\n`) + } + + const instance: DevInstance = { + info, + paths, + abortSignal: abortController.signal, + update: async (changes) => { + if (releasePromise || compromised) return + if (changes.status) info.status = changes.status + if (changes.details) info.details = {...info.details, ...changes.details} + info.updatedAt = formatJsonOutputTimestamp(new Date()) + const snapshot = {...info, details: {...info.details}} + const write = updates.then(async () => { + if (!compromised) await writeDevInstanceInfo(paths.infoFile, snapshot) + }) + updates = write.catch(() => {}) + await write + }, + log: writeLogEntry, + tee: (target, {source, stream}) => { + const decoder = new StringDecoder('utf8') + let pending = '' + const flushLines = async (text: string) => { + const lines = text.split(/\r?\n/) + pending = lines.pop() ?? '' + await Promise.all( + lines.map(async (line) => { + const message = unstyled(line).trimEnd() + if (message.length > 0) await writeLogEntry({source, stream, message}) + }), + ) + } + const flush = () => flushLines(`${pending}${decoder.end()}\n`) + const output = new Writable({ + write(chunk: Buffer | string, encoding, callback) { + Promise.all([ + writeToStream(target, chunk, encoding), + flushLines(pending + (typeof chunk === 'string' ? chunk : decoder.write(chunk))), + ]) + .then(() => callback()) + .catch(callback) + }, + final(callback) { + flush() + .then(() => callback()) + .catch(callback) + }, + }) + const onTargetError = (error: Error) => output.destroy(error) + target.on('error', onTargetError) + output.on('error', (error) => abortController.abort(error)) + output.once('close', () => { + target.off('error', onTargetError) + }) + outputs.add(output) + return output + }, + release: () => { + releasePromise ??= (async () => { + try { + const results = await Promise.allSettled([...outputs].map((output) => endStream(output))) + const failure = results.find((result) => result.status === 'rejected') + if (failure?.status === 'rejected') throw failure.reason + await updates + } finally { + released = true + process.removeListener('exit', removeInfoOnExit) + try { + await endStream(logStream) + } finally { + const owned = ownsInfo() + await releaseLock().catch((error: Error) => + outputDebug(`Failed to release the dev instance lock: ${error.message}`), + ) + if (owned) await removeFile(paths.infoFile).catch(() => {}) + } + } + })() + return releasePromise + }, + } + + try { + await once(logStream, 'open') + await writeDevInstanceInfo(paths.infoFile, info) + published = true + return instance + } catch (error) { + // Closing an already failed log can reject too; preserve the original setup error. + await instance.release().catch((cleanupError: Error) => outputDebug(cleanupError.message)) + await removeFile(`${paths.infoFile}.${sessionId}.tmp`) + throw error + } +} + +async function writeToStream(stream: Writable, chunk: Buffer | string, encoding: BufferEncoding = 'utf8') { + await new Promise((resolve, reject) => { + stream.write(chunk, encoding, (error) => (error ? reject(error) : resolve())) + }) +} + +async function endStream(stream: Writable) { + const completed = finished(stream, {cleanup: true}) + stream.end() + await completed +} + +/** + * Returns the running instance of a command for a project, if there is one. + * + * @param identity - The command, directory and optional key of the instance. + * @returns The instance description, or undefined when no live instance holds the lock. + */ +export async function readDevInstance(identity: DevInstanceIdentity): Promise { + const paths = devInstancePaths(identity) + return readLiveDevInstance(paths) +} + +/** + * Lists every running dev instance of a project, across commands and keys. + * + * @param directory - The project directory. + * @returns The descriptions of the live instances. + */ +export async function listDevInstances(directory: string): Promise { + const folder = joinPath(directory, '.shopify', DEV_INSTANCES_FOLDER) + if (!(await fileExists(folder))) return [] + const infoFiles = await glob('*.json', {cwd: folder, absolute: true, onlyFiles: true}) + const instances = await Promise.all( + infoFiles.map((infoFile) => + readLiveDevInstance({ + folder, + infoFile, + lockFile: infoFile.replace(/\.json$/, '.lock'), + logFile: infoFile.replace(/\.json$/, '.jsonl'), + }), + ), + ) + return instances.filter((instance): instance is DevInstanceInfo => instance !== undefined) +} + +/** + * Checks whether a process exists. A process owned by another user counts as alive. + * + * @param pid - The process ID. + * @returns True when the process exists. + */ +export function isProcessAlive(pid: number): boolean { + if (!Number.isInteger(pid) || pid <= 0) return false + try { + process.kill(pid, 0) + return true + // eslint-disable-next-line no-catch-all/no-catch-all + } catch (error) { + return (error as NodeJS.ErrnoException).code === 'EPERM' + } +} + +async function readLiveDevInstance(paths: DevInstancePaths): Promise { + const info = await readDevInstanceInfo(paths.infoFile) + if (!info || !isProcessAlive(info.pid)) return undefined + if (!(await lockIsFresh(paths))) return undefined + return info +} + +async function lockIsFresh(paths: DevInstancePaths): Promise { + try { + return await lockfile.check(paths.infoFile, { + realpath: false, + lockfilePath: paths.lockFile, + stale: DEFAULT_STALE_MS, + }) + // eslint-disable-next-line no-catch-all/no-catch-all + } catch { + return false + } +} + +async function readDevInstanceInfo(infoFile: string): Promise { + try { + const parsed = JSON.parse(await readFile(infoFile)) as DevInstanceInfo + return typeof parsed?.pid === 'number' ? parsed : undefined + // eslint-disable-next-line no-catch-all/no-catch-all + } catch { + return undefined + } +} + +async function writeDevInstanceInfo(infoFile: string, info: DevInstanceInfo): Promise { + // Write then rename so readers never see a partial file. Details may hold local URLs with keys, so keep it private. + const temporaryFile = `${infoFile}.${info.sessionId}.tmp` + await writeFileWithMode(temporaryFile, `${JSON.stringify(info, null, 2)}\n`, {mode: 0o600}) + await renameFile(temporaryFile, infoFile) +} + +function isLockedError(error: unknown): boolean { + return (error as NodeJS.ErrnoException | undefined)?.code === 'ELOCKED' +} + +function devInstanceSlug({command, key}: DevInstanceIdentity): string { + return [command, key] + .filter(Boolean) + .join('-') + .toLowerCase() + .replace(/[^a-z0-9._-]+/g, '-') + .replace(/^-+|-+$/g, '') +} + +function devInstanceAlreadyRunningError( + identity: DevInstanceIdentity, + existing: DevInstanceInfo | undefined, +): AbortError { + const command = `shopify ${identity.command}` + if (!existing) { + const error = new AbortError( + `Another \`${command}\` is starting for this project.`, + 'Wait a few seconds and try again, or stop the other process first.', + ) + error.code = DEV_INSTANCE_ALREADY_RUNNING_CODE + return error + } + + const stopCommand = process.platform === 'win32' ? `taskkill /PID ${existing.pid} /F` : `kill ${existing.pid}` + const detailLines = Object.entries(existing.details) + .filter(([, value]) => typeof value === 'string' || typeof value === 'number' || typeof value === 'boolean') + .map(([name, value]) => [name, String(value)]) + + const error = new AbortError( + `Another \`${command}\` is already running for this project.`, + 'Use the running instance, or stop it before starting a new one.', + [ + ['Read its output in', {filePath: existing.logFile}], + ['Stop it with', {command: stopCommand}], + ], + [ + { + body: { + tabularData: [['PID', String(existing.pid)], ['Started', existing.startedAt], ...detailLines], + firstColumnSubdued: true, + }, + }, + ], + ) + error.code = DEV_INSTANCE_ALREADY_RUNNING_CODE + error.details = {instance: existing} + return error +} diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 7a1f983b4f9..6671b1afa94 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -448,6 +448,9 @@ importers: pathe: specifier: 1.1.2 version: 1.1.2 + proper-lockfile: + specifier: 4.1.2 + version: 4.1.2 react: specifier: 19.2.4 version: 19.2.4 @@ -491,6 +494,9 @@ importers: '@types/lodash': specifier: 4.17.24 version: 4.17.24 + '@types/proper-lockfile': + specifier: 4.1.4 + version: 4.1.4 '@types/react': specifier: 18.3.12 version: 18.3.12 From 101d5beb05cda713d60b26a27a811ca63d6f7ede Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Isaac=20Rold=C3=A1n?= Date: Mon, 19 Oct 2026 17:58:11 +0200 Subject: [PATCH 02/14] Omit session IDs from dev log records --- packages/cli-kit/src/public/node/dev-instance.test.ts | 6 +++--- packages/cli-kit/src/public/node/dev-instance.ts | 6 ++---- 2 files changed, 5 insertions(+), 7 deletions(-) diff --git a/packages/cli-kit/src/public/node/dev-instance.test.ts b/packages/cli-kit/src/public/node/dev-instance.test.ts index 8a0d8c647fb..4879f9b487a 100644 --- a/packages/cli-kit/src/public/node/dev-instance.test.ts +++ b/packages/cli-kit/src/public/node/dev-instance.test.ts @@ -50,6 +50,7 @@ describe('acquireDevInstance', () => { const info = JSON.parse(await readFile(paths.infoFile)) expect(info).toMatchObject({ schemaVersion: 1, + sessionId: instance.info.sessionId, command: 'app dev', key: 'shopify.app.toml', pid: process.pid, @@ -292,12 +293,11 @@ describe('DevInstance log', () => { // Then expect(chunks.join('')).toBe('\u001b[32mready\u001b[39m in 20ms\npartial line\n\n') const entries = await readLog(instance.paths.logFile) - expect(entries.map(({sessionId, ...entry}) => entry)).toEqual([ + expect(entries).toEqual([ {timestamp: expect.any(String), source: 'web', stream: 'stdout', message: 'ready in 20ms'}, {timestamp: expect.any(String), source: 'web', stream: 'stdout', message: 'partial line'}, {timestamp: expect.any(String), source: 'app dev', level: 'info', message: 'Preview URL: https://preview'}, ]) - expect(entries.every((entry) => entry.sessionId === instance.info.sessionId)).toBe(true) expect(entries[0].timestamp).toMatch(/^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}Z$/) }) }) @@ -311,7 +311,7 @@ describe('DevInstance log', () => { output.write(bytes.subarray(4)) await instance.release() await expect(readLog(instance.paths.logFile)).resolves.toEqual([ - expect.objectContaining({sessionId: instance.info.sessionId, source: 'web', message: 'café'}), + {timestamp: expect.any(String), source: 'web', stream: 'stdout', message: 'café'}, ]) }) }) diff --git a/packages/cli-kit/src/public/node/dev-instance.ts b/packages/cli-kit/src/public/node/dev-instance.ts index 346322a4f06..0105eadb0ce 100644 --- a/packages/cli-kit/src/public/node/dev-instance.ts +++ b/packages/cli-kit/src/public/node/dev-instance.ts @@ -48,7 +48,6 @@ export interface DevInstanceInfo { /** One line of a dev instance's JSON Lines log. */ export interface DevInstanceLogEntry { - sessionId: string timestamp: string /** What produced the line, for example a process prefix like `web` or `extensions`. */ source: string @@ -97,7 +96,7 @@ export interface DevInstance { /** Updates and persists the instance description. */ update: (changes: DevInstanceUpdate) => Promise /** Appends one entry to the JSON Lines log. */ - log: (entry: Omit) => Promise + log: (entry: Omit) => Promise /** * Wraps a stream so every line written to it is also appended to the log, without ANSI styling. * The returned stream forwards the original bytes to `target` unchanged. @@ -249,10 +248,9 @@ async function createDevInstance(options: AcquireDevInstanceOptions, paths: DevI } process.once('exit', removeInfoOnExit) - const writeLogEntry = async (entry: Omit) => { + const writeLogEntry = async (entry: Omit) => { if (released || compromised) return const line: DevInstanceLogEntry = { - sessionId: info.sessionId, timestamp: formatJsonOutputTimestamp(new Date()), ...entry, } From d9f1fe50f39800af9c3f71e22472a30b846744a1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Isaac=20Rold=C3=A1n?= Date: Tue, 20 Oct 2026 10:18:19 +0200 Subject: [PATCH 03/14] Preserve output sources in dev session logs and queued writes --- .../node/ui/components/ConcurrentOutput.tsx | 5 ++ .../src/public/node/dev-instance.test.ts | 59 +++++++++++++++++++ .../cli-kit/src/public/node/dev-instance.ts | 52 +++++++++++++++- 3 files changed, 113 insertions(+), 3 deletions(-) diff --git a/packages/cli-kit/src/private/node/ui/components/ConcurrentOutput.tsx b/packages/cli-kit/src/private/node/ui/components/ConcurrentOutput.tsx index 081b16667ff..6e5e59c0f68 100644 --- a/packages/cli-kit/src/private/node/ui/components/ConcurrentOutput.tsx +++ b/packages/cli-kit/src/private/node/ui/components/ConcurrentOutput.tsx @@ -57,6 +57,11 @@ function useConcurrentOutputContext(context: ConcurrentOutputContext, callbac return outputContextStore.run(context, callback) } +/** Returns the source and styling context for the current output write. */ +export function getConcurrentOutputContext(): ConcurrentOutputContext | undefined { + return outputContextStore.getStore() +} + /** Runs finite processes concurrently and routes their output through the shared diagnostic context. */ export async function runConcurrentProcessesForJson({ processes, diff --git a/packages/cli-kit/src/public/node/dev-instance.test.ts b/packages/cli-kit/src/public/node/dev-instance.test.ts index 4879f9b487a..36ef872ccdd 100644 --- a/packages/cli-kit/src/public/node/dev-instance.test.ts +++ b/packages/cli-kit/src/public/node/dev-instance.test.ts @@ -8,6 +8,10 @@ import { } from './dev-instance.js' import {AbortError} from './error.js' import {fileExists, inTemporaryDirectory, mkdir, readFile, rmdir, writeFile} from './fs.js' +import { + getConcurrentOutputContext, + useConcurrentOutputContext, +} from '../../private/node/ui/components/ConcurrentOutput.js' import {describe, expect, test, vi} from 'vitest' import {Writable} from 'stream' import {utimes} from 'fs/promises' @@ -276,6 +280,61 @@ describe('acquireDevInstance', () => { }) describe('DevInstance log', () => { + test('preserves extension sources in queued terminal writes and log records', async () => { + await inTemporaryDirectory(async (directory) => { + const instance = await acquireDevInstance({command: 'app dev', directory}) + const terminal: {source: string | undefined; message: string}[] = [] + let resume: () => void = () => {} + const target = new Writable({ + write(chunk, _encoding, callback) { + terminal.push({source: getConcurrentOutputContext()?.outputPrefix, message: chunk.toString()}) + if (terminal.length === 1) resume = callback + else callback() + }, + }) + const output = instance.tee(target, {source: 'app-preview', stream: 'stdout'}) + try { + useConcurrentOutputContext({outputPrefix: 'discount'}, () => { + output.write('Building discount...\n') + output.write('Finished discount\n') + }) + useConcurrentOutputContext({outputPrefix: 'admin-action'}, () => output.write('Bundling action...\n')) + useConcurrentOutputContext({outputPrefix: 'app-config'}, () => output.end('Configuration accepted')) + resume() + await instance.release() + expect(terminal).toEqual([ + {source: 'discount', message: 'Building discount...\n'}, + {source: 'discount', message: 'Finished discount\n'}, + {source: 'admin-action', message: 'Bundling action...\n'}, + {source: 'app-config', message: 'Configuration accepted'}, + ]) + const entries = await readLog(instance.paths.logFile) + expect(entries.map(({source, message}) => ({source, message}))).toEqual([ + {source: 'discount', message: 'Building discount...'}, + {source: 'discount', message: 'Finished discount'}, + {source: 'admin-action', message: 'Bundling action...'}, + {source: 'app-config', message: 'Configuration accepted'}, + ]) + } finally { + await instance.release() + } + }) + }) + + test('keeps incomplete messages from different sources separate', async () => { + await inTemporaryDirectory(async (directory) => { + const instance = await acquireDevInstance({command: 'app dev', directory}) + const output = instance.tee(collectingStream().stream, {source: 'app-preview', stream: 'stdout'}) + useConcurrentOutputContext({outputPrefix: 'discount'}, () => output.write('first')) + useConcurrentOutputContext({outputPrefix: 'admin-action'}, () => output.write('second')) + await instance.release() + expect((await readLog(instance.paths.logFile)).map(({source, message}) => ({source, message}))).toEqual([ + {source: 'discount', message: 'first'}, + {source: 'admin-action', message: 'second'}, + ]) + }) + }) + test('tees process output into a JSON Lines log without ANSI styling', async () => { await inTemporaryDirectory(async (directory) => { // Given diff --git a/packages/cli-kit/src/public/node/dev-instance.ts b/packages/cli-kit/src/public/node/dev-instance.ts index 0105eadb0ce..9065212cb4b 100644 --- a/packages/cli-kit/src/public/node/dev-instance.ts +++ b/packages/cli-kit/src/public/node/dev-instance.ts @@ -5,10 +5,15 @@ import {AbortError} from './error.js' import {outputDebug, unstyled} from './output.js' import {formatJsonOutputTimestamp} from './json-output-schema.js' import {randomUUID} from './crypto.js' +import { + ConcurrentOutputContext, + getConcurrentOutputContext, + useConcurrentOutputContext, +} from '../../private/node/ui/components/ConcurrentOutput.js' import lockfile from 'proper-lockfile' import fs, {createWriteStream, readFileSync, type WriteStream} from 'fs' import {mkdir as mkdirExclusive, rmdir as removeEmptyDirectory, writeFile as writeFileWithMode} from 'fs/promises' -import {Writable} from 'stream' +import {Writable, type WritableOptions} from 'stream' import {finished} from 'stream/promises' import {StringDecoder} from 'string_decoder' import {once} from 'events' @@ -277,20 +282,25 @@ async function createDevInstance(options: AcquireDevInstanceOptions, paths: DevI tee: (target, {source, stream}) => { const decoder = new StringDecoder('utf8') let pending = '' + let currentSource = source const flushLines = async (text: string) => { const lines = text.split(/\r?\n/) pending = lines.pop() ?? '' await Promise.all( lines.map(async (line) => { const message = unstyled(line).trimEnd() - if (message.length > 0) await writeLogEntry({source, stream, message}) + if (message.length > 0) await writeLogEntry({source: currentSource, stream, message}) }), ) } const flush = () => flushLines(`${pending}${decoder.end()}\n`) - const output = new Writable({ + const output = new DevInstanceOutput({ write(chunk: Buffer | string, encoding, callback) { + const nextSource = getConcurrentOutputContext()?.outputPrefix ?? source + const previous = nextSource === currentSource ? Promise.resolve() : flush() + currentSource = nextSource Promise.all([ + previous, writeToStream(target, chunk, encoding), flushLines(pending + (typeof chunk === 'string' ? chunk : decoder.write(chunk))), ]) @@ -350,6 +360,42 @@ async function createDevInstance(options: AcquireDevInstanceOptions, paths: DevI } } +// Writable processes queued chunks in an earlier write's context. Capture context when each chunk is submitted. +class DevInstanceOutput extends Writable { + private readonly contexts: (ConcurrentOutputContext | undefined)[] = [] + + constructor(options: WritableOptions & {write: NonNullable}) { + super({ + ...options, + write: (chunk, encoding, callback) => { + useConcurrentOutputContext(this.contexts.shift() ?? {}, () => + options.write.call(this, chunk, encoding, callback), + ) + }, + }) + } + + write( + chunk: string | Uint8Array, + encoding?: BufferEncoding | ((error?: Error | null) => void), + callback?: (error?: Error | null) => void, + ): boolean { + this.contexts.push(getConcurrentOutputContext()) + return typeof encoding === 'string' ? super.write(chunk, encoding, callback) : super.write(chunk, encoding) + } + + end( + chunk?: string | Uint8Array | null | (() => void), + encoding?: BufferEncoding | (() => void), + callback?: () => void, + ): this { + if (chunk !== undefined && chunk !== null && typeof chunk !== 'function') + this.contexts.push(getConcurrentOutputContext()) + if (typeof chunk === 'function') return super.end(chunk) + return typeof encoding === 'string' ? super.end(chunk, encoding, callback) : super.end(chunk, encoding) + } +} + async function writeToStream(stream: Writable, chunk: Buffer | string, encoding: BufferEncoding = 'utf8') { await new Promise((resolve, reject) => { stream.write(chunk, encoding, (error) => (error ? reject(error) : resolve())) From 29f003732be0bd40ead66bf3afc93312db6e8def Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Isaac=20Rold=C3=A1n?= Date: Tue, 20 Oct 2026 10:22:31 +0200 Subject: [PATCH 04/14] Capture complete CLI messages without requiring line endings --- .../src/public/node/dev-instance.test.ts | 37 +++++++++++++++++++ .../cli-kit/src/public/node/dev-instance.ts | 20 ++++++++-- 2 files changed, 53 insertions(+), 4 deletions(-) diff --git a/packages/cli-kit/src/public/node/dev-instance.test.ts b/packages/cli-kit/src/public/node/dev-instance.test.ts index 36ef872ccdd..5d45b60aeb2 100644 --- a/packages/cli-kit/src/public/node/dev-instance.test.ts +++ b/packages/cli-kit/src/public/node/dev-instance.test.ts @@ -280,6 +280,43 @@ describe('acquireDevInstance', () => { }) describe('DevInstance log', () => { + test('records complete string messages immediately while assembling byte output into lines', async () => { + await inTemporaryDirectory(async (directory) => { + const instance = await acquireDevInstance({command: 'app dev', directory}) + const chunks: Buffer[] = [] + const target = new Writable({ + write(chunk, _encoding, callback) { + chunks.push(Buffer.from(chunk)) + callback() + }, + }) + const output = instance.tee(target, {source: 'web', stream: 'stdout', stringWritesAreMessages: true}) + const write = (chunk: string | Buffer) => + new Promise((resolve, reject) => { + output.write(chunk, (error) => (error ? reject(error) : resolve())) + }) + try { + await write('Bundling extension...') + await expect(readLog(instance.paths.logFile)).resolves.toEqual([ + {timestamp: expect.any(String), source: 'web', stream: 'stdout', message: 'Bundling extension...'}, + ]) + await write('Extension built') + const bytes = Buffer.from('café\n') + await write(bytes.subarray(0, 4)) + await write(bytes.subarray(4)) + await instance.release() + expect(Buffer.concat(chunks).toString()).toBe('Bundling extension...Extension builtcafé\n') + expect((await readLog(instance.paths.logFile)).map(({message}) => message)).toEqual([ + 'Bundling extension...', + 'Extension built', + 'café', + ]) + } finally { + await instance.release() + } + }) + }) + test('preserves extension sources in queued terminal writes and log records', async () => { await inTemporaryDirectory(async (directory) => { const instance = await acquireDevInstance({command: 'app dev', directory}) diff --git a/packages/cli-kit/src/public/node/dev-instance.ts b/packages/cli-kit/src/public/node/dev-instance.ts index 9065212cb4b..25cd7cd3488 100644 --- a/packages/cli-kit/src/public/node/dev-instance.ts +++ b/packages/cli-kit/src/public/node/dev-instance.ts @@ -106,7 +106,15 @@ export interface DevInstance { * Wraps a stream so every line written to it is also appended to the log, without ANSI styling. * The returned stream forwards the original bytes to `target` unchanged. */ - tee: (target: Writable, options: {source: string; stream: 'stdout' | 'stderr'}) => Writable + tee: ( + target: Writable, + options: { + source: string + stream: 'stdout' | 'stderr' + /** Treat each string write as a complete message. Byte writes remain buffered into lines. */ + stringWritesAreMessages?: boolean + }, + ) => Writable /** Closes the log, removes the instance description and releases the lock. Safe to call more than once. */ release: () => Promise } @@ -279,7 +287,7 @@ async function createDevInstance(options: AcquireDevInstanceOptions, paths: DevI await write }, log: writeLogEntry, - tee: (target, {source, stream}) => { + tee: (target, {source, stream, stringWritesAreMessages = false}) => { const decoder = new StringDecoder('utf8') let pending = '' let currentSource = source @@ -295,14 +303,18 @@ async function createDevInstance(options: AcquireDevInstanceOptions, paths: DevI } const flush = () => flushLines(`${pending}${decoder.end()}\n`) const output = new DevInstanceOutput({ + decodeStrings: !stringWritesAreMessages, write(chunk: Buffer | string, encoding, callback) { const nextSource = getConcurrentOutputContext()?.outputPrefix ?? source - const previous = nextSource === currentSource ? Promise.resolve() : flush() + const completeMessage = stringWritesAreMessages && typeof chunk === 'string' + const previous = nextSource !== currentSource || completeMessage ? flush() : Promise.resolve() currentSource = nextSource Promise.all([ previous, writeToStream(target, chunk, encoding), - flushLines(pending + (typeof chunk === 'string' ? chunk : decoder.write(chunk))), + flushLines( + pending + (typeof chunk === 'string' ? chunk : decoder.write(chunk)) + (completeMessage ? '\n' : ''), + ), ]) .then(() => callback()) .catch(callback) From 81342edcf8f2ccb0d95cc4a0899f01921250c704 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Isaac=20Rold=C3=A1n?= Date: Tue, 20 Oct 2026 12:30:35 +0200 Subject: [PATCH 05/14] Complete the shared dev instance JSONL writer and schema --- .../src/public/node/dev-instance.test.ts | 63 +++++++++++++++---- .../cli-kit/src/public/node/dev-instance.ts | 58 +++++++++++------ 2 files changed, 91 insertions(+), 30 deletions(-) diff --git a/packages/cli-kit/src/public/node/dev-instance.test.ts b/packages/cli-kit/src/public/node/dev-instance.test.ts index 5d45b60aeb2..e16c3f07a51 100644 --- a/packages/cli-kit/src/public/node/dev-instance.test.ts +++ b/packages/cli-kit/src/public/node/dev-instance.test.ts @@ -38,6 +38,47 @@ async function readLog(path: string) { } describe('acquireDevInstance', () => { + test('writes identical JSONL bytes to the log and an output stream without changing supplied timestamps', async () => { + await inTemporaryDirectory(async (directory) => { + const output = collectingStream() + const instance = await acquireDevInstance({command: 'app dev', directory, logOutput: output.stream}) + await Promise.all([ + instance.log({timestamp: '2026-10-20T09:00:00Z', source: 'app dev', level: 'info', message: 'Setup'}), + instance.log({source: 'web', message: 'Warning'}), + ]) + await instance.release() + expect(output.chunks.join('')).toBe(await readFile(instance.paths.logFile)) + expect(JSON.parse(output.chunks[0]!)).toEqual({ + timestamp: '2026-10-20T09:00:00Z', + source: 'app dev', + level: 'info', + message: 'Setup', + }) + expect(output.stream.writableEnded).toBe(false) + }) + }) + + test('aborts and releases ownership when the mirror fails', async () => { + await inTemporaryDirectory(async (directory) => { + let finishWrite: ((error?: Error | null) => void) | undefined + const output = new Writable({ + write: (_chunk, _encoding, callback) => { + finishWrite = callback + }, + }) + const instance = await acquireDevInstance({command: 'app dev', directory, logOutput: output}) + const failure = new Error('Output failed') + const writing = instance.log({source: 'web', message: 'Server ready'}) + const rejected = expect(writing).rejects.toBe(failure) + await vi.waitFor(() => expect(finishWrite).toBeDefined()) + finishWrite?.(failure) + await rejected + expect(instance.abortSignal.aborted).toBe(true) + await instance.release() + await expect(readDevInstance({command: 'app dev', directory})).resolves.toBeUndefined() + }) + }) + test('describes the instance in .shopify/dev and releases it', async () => { await inTemporaryDirectory(async (directory) => { // When @@ -290,7 +331,7 @@ describe('DevInstance log', () => { callback() }, }) - const output = instance.tee(target, {source: 'web', stream: 'stdout', stringWritesAreMessages: true}) + const output = instance.tee(target, {source: 'web', stringWritesAreMessages: true}) const write = (chunk: string | Buffer) => new Promise((resolve, reject) => { output.write(chunk, (error) => (error ? reject(error) : resolve())) @@ -298,7 +339,7 @@ describe('DevInstance log', () => { try { await write('Bundling extension...') await expect(readLog(instance.paths.logFile)).resolves.toEqual([ - {timestamp: expect.any(String), source: 'web', stream: 'stdout', message: 'Bundling extension...'}, + {timestamp: expect.any(String), source: 'web', message: 'Bundling extension...'}, ]) await write('Extension built') const bytes = Buffer.from('café\n') @@ -329,7 +370,7 @@ describe('DevInstance log', () => { else callback() }, }) - const output = instance.tee(target, {source: 'app-preview', stream: 'stdout'}) + const output = instance.tee(target, {source: 'app-preview'}) try { useConcurrentOutputContext({outputPrefix: 'discount'}, () => { output.write('Building discount...\n') @@ -361,7 +402,7 @@ describe('DevInstance log', () => { test('keeps incomplete messages from different sources separate', async () => { await inTemporaryDirectory(async (directory) => { const instance = await acquireDevInstance({command: 'app dev', directory}) - const output = instance.tee(collectingStream().stream, {source: 'app-preview', stream: 'stdout'}) + const output = instance.tee(collectingStream().stream, {source: 'app-preview'}) useConcurrentOutputContext({outputPrefix: 'discount'}, () => output.write('first')) useConcurrentOutputContext({outputPrefix: 'admin-action'}, () => output.write('second')) await instance.release() @@ -377,7 +418,7 @@ describe('DevInstance log', () => { // Given const instance = await acquireDevInstance({command: 'app dev', directory}) const {stream: target, chunks} = collectingStream() - const stdout = instance.tee(target, {source: 'web', stream: 'stdout'}) + const stdout = instance.tee(target, {source: 'web'}) // When stdout.write('\u001b[32mready\u001b[39m in 20ms\npartial ') @@ -390,8 +431,8 @@ describe('DevInstance log', () => { expect(chunks.join('')).toBe('\u001b[32mready\u001b[39m in 20ms\npartial line\n\n') const entries = await readLog(instance.paths.logFile) expect(entries).toEqual([ - {timestamp: expect.any(String), source: 'web', stream: 'stdout', message: 'ready in 20ms'}, - {timestamp: expect.any(String), source: 'web', stream: 'stdout', message: 'partial line'}, + {timestamp: expect.any(String), source: 'web', message: 'ready in 20ms'}, + {timestamp: expect.any(String), source: 'web', message: 'partial line'}, {timestamp: expect.any(String), source: 'app dev', level: 'info', message: 'Preview URL: https://preview'}, ]) expect(entries[0].timestamp).toMatch(/^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}Z$/) @@ -401,13 +442,13 @@ describe('DevInstance log', () => { test('flushes a split UTF-8 line on release without requiring the source to end', async () => { await inTemporaryDirectory(async (directory) => { const instance = await acquireDevInstance({command: 'app dev', directory}) - const output = instance.tee(collectingStream().stream, {source: 'web', stream: 'stdout'}) + const output = instance.tee(collectingStream().stream, {source: 'web'}) const bytes = Buffer.from('café') output.write(bytes.subarray(0, 4)) output.write(bytes.subarray(4)) await instance.release() await expect(readLog(instance.paths.logFile)).resolves.toEqual([ - {timestamp: expect.any(String), source: 'web', stream: 'stdout', message: 'café'}, + {timestamp: expect.any(String), source: 'web', message: 'café'}, ]) }) }) @@ -443,7 +484,7 @@ describe('DevInstance log', () => { } }, }) - const output = instance.tee(target, {source: 'web', stream: 'stdout'}) + const output = instance.tee(target, {source: 'web'}) let written = false const completed = new Promise((resolve, reject) => { const available = output.write(`${'x'.repeat(output.writableHighWaterMark)}\n`, (error) => { @@ -475,7 +516,7 @@ describe('DevInstance log', () => { callback(new Error('Output failed')) }, }) - const output = instance.tee(target, {source: 'web', stream: 'stdout'}) + const output = instance.tee(target, {source: 'web'}) await expect( new Promise((resolve, reject) => { output.write('message\n', (error) => (error ? reject(error) : resolve())) diff --git a/packages/cli-kit/src/public/node/dev-instance.ts b/packages/cli-kit/src/public/node/dev-instance.ts index 25cd7cd3488..805ebd8b9e7 100644 --- a/packages/cli-kit/src/public/node/dev-instance.ts +++ b/packages/cli-kit/src/public/node/dev-instance.ts @@ -3,7 +3,9 @@ import {joinPath} from './path.js' import {fileExists, glob, mkdir, readFile, removeFile, removeFileSync, renameFile, rmdir} from './fs.js' import {AbortError} from './error.js' import {outputDebug, unstyled} from './output.js' -import {formatJsonOutputTimestamp} from './json-output-schema.js' +import {defineJsonOutputSchema, formatJsonOutputTimestamp, jsonOutputTimestampSchema} from './json-output-schema.js' +import {jsonErrorOutputSchema, JsonErrorSchema} from './error/schema.js' +import {zod} from './schema.js' import {randomUUID} from './crypto.js' import { ConcurrentOutputContext, @@ -51,16 +53,22 @@ export interface DevInstanceInfo { details: {[key: string]: unknown} } +export const devInstanceLogOutputSchema = defineJsonOutputSchema({ + name: 'DevInstanceLogEntry', + schema: zod + .object({ + timestamp: jsonOutputTimestampSchema, + source: zod.string().describe('What produced the line, for example a process prefix like web or extensions.'), + level: zod.enum(['debug', 'info', 'warning', 'error']).optional(), + message: zod.string(), + error: JsonErrorSchema.optional().describe('The shared CLI fatal error, included when the session fails.'), + }) + .strict(), + definitions: jsonErrorOutputSchema.definitions, +}) + /** One line of a dev instance's JSON Lines log. */ -export interface DevInstanceLogEntry { - timestamp: string - /** What produced the line, for example a process prefix like `web` or `extensions`. */ - source: string - /** The stream the line was written to, when it comes from a process. */ - stream?: 'stdout' | 'stderr' - level?: 'debug' | 'info' | 'warning' | 'error' - message: string -} +export type DevInstanceLogEntry = zod.infer export interface DevInstancePaths { /** The folder that holds every dev instance of the project: `.shopify/dev`. */ @@ -85,6 +93,8 @@ export interface DevInstanceIdentity { export interface AcquireDevInstanceOptions extends DevInstanceIdentity { /** Initial command-specific details. */ details?: {[key: string]: unknown} + /** Receives the exact JSONL bytes written to the session log. The caller owns this stream. */ + logOutput?: Writable } export interface DevInstanceUpdate { @@ -101,7 +111,7 @@ export interface DevInstance { /** Updates and persists the instance description. */ update: (changes: DevInstanceUpdate) => Promise /** Appends one entry to the JSON Lines log. */ - log: (entry: Omit) => Promise + log: (entry: Omit & {timestamp?: string}) => Promise /** * Wraps a stream so every line written to it is also appended to the log, without ANSI styling. * The returned stream forwards the original bytes to `target` unchanged. @@ -110,7 +120,6 @@ export interface DevInstance { target: Writable, options: { source: string - stream: 'stdout' | 'stderr' /** Treat each string write as a complete message. Byte writes remain buffered into lines. */ stringWritesAreMessages?: boolean }, @@ -246,7 +255,9 @@ async function createDevInstance(options: AcquireDevInstanceOptions, paths: DevI // Each run starts a fresh log. const logStream: WriteStream = createWriteStream(paths.logFile, {flags: 'w', mode: 0o600}) - logStream.on('error', (error) => abortController.abort(error)) + const onOutputError = (error: Error) => abortController.abort(error) + logStream.on('error', onOutputError) + options.logOutput?.on('error', onOutputError) let updates = Promise.resolve() const outputs = new Set() @@ -261,13 +272,20 @@ async function createDevInstance(options: AcquireDevInstanceOptions, paths: DevI } process.once('exit', removeInfoOnExit) - const writeLogEntry = async (entry: Omit) => { + const writeLogEntry: DevInstance['log'] = async (entry) => { if (released || compromised) return - const line: DevInstanceLogEntry = { - timestamp: formatJsonOutputTimestamp(new Date()), + const record = devInstanceLogOutputSchema.validate({ ...entry, + timestamp: entry.timestamp ?? formatJsonOutputTimestamp(new Date()), + }) + const line = `${JSON.stringify(record)}\n` + try { + await writeToStream(logStream, line) + if (options.logOutput) await writeToStream(options.logOutput, line) + } catch (error) { + abortController.abort(error) + throw error } - await writeToStream(logStream, `${JSON.stringify(line)}\n`) } const instance: DevInstance = { @@ -287,7 +305,7 @@ async function createDevInstance(options: AcquireDevInstanceOptions, paths: DevI await write }, log: writeLogEntry, - tee: (target, {source, stream, stringWritesAreMessages = false}) => { + tee: (target, {source, stringWritesAreMessages = false}) => { const decoder = new StringDecoder('utf8') let pending = '' let currentSource = source @@ -297,7 +315,7 @@ async function createDevInstance(options: AcquireDevInstanceOptions, paths: DevI await Promise.all( lines.map(async (line) => { const message = unstyled(line).trimEnd() - if (message.length > 0) await writeLogEntry({source: currentSource, stream, message}) + if (message.length > 0) await writeLogEntry({source: currentSource, message}) }), ) } @@ -346,7 +364,9 @@ async function createDevInstance(options: AcquireDevInstanceOptions, paths: DevI process.removeListener('exit', removeInfoOnExit) try { await endStream(logStream) + if (options.logOutput && !options.logOutput.destroyed) await writeToStream(options.logOutput, '') } finally { + options.logOutput?.off('error', onOutputError) const owned = ownsInfo() await releaseLock().catch((error: Error) => outputDebug(`Failed to release the dev instance lock: ${error.message}`), From 42456808a62c08f2b08546debce152118bc2eb91 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Isaac=20Rold=C3=A1n?= Date: Tue, 20 Oct 2026 13:29:38 +0200 Subject: [PATCH 06/14] Keep CLI fatal errors outside the dev log schema --- .../cli-kit/src/public/node/dev-instance.test.ts | 12 ++++++++++++ packages/cli-kit/src/public/node/dev-instance.ts | 3 --- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/packages/cli-kit/src/public/node/dev-instance.test.ts b/packages/cli-kit/src/public/node/dev-instance.test.ts index e16c3f07a51..9f6b3aa9db9 100644 --- a/packages/cli-kit/src/public/node/dev-instance.test.ts +++ b/packages/cli-kit/src/public/node/dev-instance.test.ts @@ -1,6 +1,7 @@ import { acquireDevInstance, DEV_INSTANCE_ALREADY_RUNNING_CODE, + devInstanceLogOutputSchema, devInstancePaths, isProcessAlive, listDevInstances, @@ -38,6 +39,17 @@ async function readLog(path: string) { } describe('acquireDevInstance', () => { + test('does not include CLI fatal errors in the session log schema', () => { + expect(() => + devInstanceLogOutputSchema.validate({ + timestamp: '2026-10-20T09:00:00Z', + source: 'app dev', + message: 'Command failed', + error: {type: 'abort', message: 'Command failed'}, + }), + ).toThrow() + }) + test('writes identical JSONL bytes to the log and an output stream without changing supplied timestamps', async () => { await inTemporaryDirectory(async (directory) => { const output = collectingStream() diff --git a/packages/cli-kit/src/public/node/dev-instance.ts b/packages/cli-kit/src/public/node/dev-instance.ts index 805ebd8b9e7..53cf1fd54bb 100644 --- a/packages/cli-kit/src/public/node/dev-instance.ts +++ b/packages/cli-kit/src/public/node/dev-instance.ts @@ -4,7 +4,6 @@ import {fileExists, glob, mkdir, readFile, removeFile, removeFileSync, renameFil import {AbortError} from './error.js' import {outputDebug, unstyled} from './output.js' import {defineJsonOutputSchema, formatJsonOutputTimestamp, jsonOutputTimestampSchema} from './json-output-schema.js' -import {jsonErrorOutputSchema, JsonErrorSchema} from './error/schema.js' import {zod} from './schema.js' import {randomUUID} from './crypto.js' import { @@ -61,10 +60,8 @@ export const devInstanceLogOutputSchema = defineJsonOutputSchema({ source: zod.string().describe('What produced the line, for example a process prefix like web or extensions.'), level: zod.enum(['debug', 'info', 'warning', 'error']).optional(), message: zod.string(), - error: JsonErrorSchema.optional().describe('The shared CLI fatal error, included when the session fails.'), }) .strict(), - definitions: jsonErrorOutputSchema.definitions, }) /** One line of a dev instance's JSON Lines log. */ From 99fff634192ae21c07d64aa5e2b2810bd76215b1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Isaac=20Rold=C3=A1n?= Date: Tue, 20 Oct 2026 16:43:33 +0200 Subject: [PATCH 07/14] Preserve dev output callbacks and string encodings --- .../src/public/node/dev-instance.test.ts | 42 +++++++++++++++++++ .../cli-kit/src/public/node/dev-instance.ts | 10 +++-- 2 files changed, 49 insertions(+), 3 deletions(-) diff --git a/packages/cli-kit/src/public/node/dev-instance.test.ts b/packages/cli-kit/src/public/node/dev-instance.test.ts index 9f6b3aa9db9..b0ba9dc297e 100644 --- a/packages/cli-kit/src/public/node/dev-instance.test.ts +++ b/packages/cli-kit/src/public/node/dev-instance.test.ts @@ -15,6 +15,7 @@ import { } from '../../private/node/ui/components/ConcurrentOutput.js' import {describe, expect, test, vi} from 'vitest' import {Writable} from 'stream' +import {finished} from 'stream/promises' import {utimes} from 'fs/promises' const DEAD_PID = 2 ** 22 + 12345 @@ -333,6 +334,47 @@ describe('acquireDevInstance', () => { }) describe('DevInstance log', () => { + test.each([false, true])('calls write and end callbacks with undefined encoding, messages=%s', async (messages) => { + await inTemporaryDirectory(async (directory) => { + const instance = await acquireDevInstance({command: 'app dev', directory}) + const target = collectingStream() + const output = instance.tee(target.stream, {source: 'web', stringWritesAreMessages: messages}) + const onWrite = vi.fn() + const onEnd = vi.fn() + try { + // Node accepts an undefined encoding at runtime, but its type overload requires a string. + Reflect.apply(output.write, output, ['First line\n', undefined, onWrite]) + Reflect.apply(output.end, output, ['Last line\n', undefined, onEnd]) + await finished(output, {cleanup: true}) + expect([onWrite.mock.calls.length, onEnd.mock.calls.length]).toEqual([1, 1]) + expect(target.chunks.join('')).toBe('First line\nLast line\n') + expect((await readLog(instance.paths.logFile)).map(({message}) => message)).toEqual(['First line', 'Last line']) + } finally { + await instance.release() + } + }) + }) + + test.each([ + {encoding: 'hex', chunk: '636166c3a90a'}, + {encoding: 'base64', chunk: 'Y2Fmw6kK'}, + ] as const)('logs decoded $encoding string messages from write and end', async ({encoding, chunk}) => { + await inTemporaryDirectory(async (directory) => { + const instance = await acquireDevInstance({command: 'app dev', directory}) + const target = collectingStream() + const output = instance.tee(target.stream, {source: 'web', stringWritesAreMessages: true}) + try { + output.write(chunk, encoding) + output.end(chunk, encoding) + await finished(output, {cleanup: true}) + expect(target.chunks.join('')).toBe('café\ncafé\n') + expect((await readLog(instance.paths.logFile)).map(({message}) => message)).toEqual(['café', 'café']) + } finally { + await instance.release() + } + }) + }) + test('records complete string messages immediately while assembling byte output into lines', async () => { await inTemporaryDirectory(async (directory) => { const instance = await acquireDevInstance({command: 'app dev', directory}) diff --git a/packages/cli-kit/src/public/node/dev-instance.ts b/packages/cli-kit/src/public/node/dev-instance.ts index 53cf1fd54bb..bd67b50f23b 100644 --- a/packages/cli-kit/src/public/node/dev-instance.ts +++ b/packages/cli-kit/src/public/node/dev-instance.ts @@ -328,7 +328,9 @@ async function createDevInstance(options: AcquireDevInstanceOptions, paths: DevI previous, writeToStream(target, chunk, encoding), flushLines( - pending + (typeof chunk === 'string' ? chunk : decoder.write(chunk)) + (completeMessage ? '\n' : ''), + pending + + decoder.write(typeof chunk === 'string' ? Buffer.from(chunk, encoding) : chunk) + + (completeMessage ? '\n' : ''), ), ]) .then(() => callback()) @@ -410,7 +412,9 @@ class DevInstanceOutput extends Writable { callback?: (error?: Error | null) => void, ): boolean { this.contexts.push(getConcurrentOutputContext()) - return typeof encoding === 'string' ? super.write(chunk, encoding, callback) : super.write(chunk, encoding) + return typeof encoding === 'string' + ? super.write(chunk, encoding, callback) + : super.write(chunk, encoding ?? callback) } end( @@ -421,7 +425,7 @@ class DevInstanceOutput extends Writable { if (chunk !== undefined && chunk !== null && typeof chunk !== 'function') this.contexts.push(getConcurrentOutputContext()) if (typeof chunk === 'function') return super.end(chunk) - return typeof encoding === 'string' ? super.end(chunk, encoding, callback) : super.end(chunk, encoding) + return typeof encoding === 'string' ? super.end(chunk, encoding, callback) : super.end(chunk, encoding ?? callback) } } From 3e2c7a8eee52f3b6b0618ad1d392aa25f98a4d58 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Isaac=20Rold=C3=A1n?= Date: Tue, 20 Oct 2026 16:46:21 +0200 Subject: [PATCH 08/14] Use a 15-second dev instance stale interval --- packages/cli-kit/src/public/node/dev-instance.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/cli-kit/src/public/node/dev-instance.ts b/packages/cli-kit/src/public/node/dev-instance.ts index bd67b50f23b..b2be6ab4dfe 100644 --- a/packages/cli-kit/src/public/node/dev-instance.ts +++ b/packages/cli-kit/src/public/node/dev-instance.ts @@ -26,7 +26,7 @@ import {once} from 'events' export const DEV_INSTANCE_ALREADY_RUNNING_CODE = 'dev_instance_already_running' const DEV_INSTANCES_FOLDER = 'dev' -const DEFAULT_STALE_MS = 30_000 +const DEFAULT_STALE_MS = 15_000 /** * Describes a running instance of a long-running command such as `app dev` or `theme dev`. From de1fa7e0cf80f1c096d1ccb45e9df316aa83001a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Isaac=20Rold=C3=A1n?= Date: Tue, 20 Oct 2026 18:18:43 +0200 Subject: [PATCH 09/14] Drain queued dev status updates before release --- .../cli-kit/src/public/node/dev-instance.test.ts | 13 +++++++++---- packages/cli-kit/src/public/node/dev-instance.ts | 2 +- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/packages/cli-kit/src/public/node/dev-instance.test.ts b/packages/cli-kit/src/public/node/dev-instance.test.ts index b0ba9dc297e..416fcb59ba9 100644 --- a/packages/cli-kit/src/public/node/dev-instance.test.ts +++ b/packages/cli-kit/src/public/node/dev-instance.test.ts @@ -562,12 +562,14 @@ describe('DevInstance log', () => { }) }) - test('reports destination errors and releases the instance after output fails', async () => { + test('drains queued status updates before releasing ownership after output fails', async () => { await inTemporaryDirectory(async (directory) => { const instance = await acquireDevInstance({command: 'app dev', directory}) + const updates = Array.from({length: 500}, (_, sequence) => instance.update({details: {sequence}})) + const failure = new Error('Output failed') const target = new Writable({ write(_chunk, _encoding, callback) { - callback(new Error('Output failed')) + callback(failure) }, }) const output = instance.tee(target, {source: 'web'}) @@ -575,11 +577,14 @@ describe('DevInstance log', () => { new Promise((resolve, reject) => { output.write('message\n', (error) => (error ? reject(error) : resolve())) }), - ).rejects.toThrow('Output failed') - await expect(instance.release()).rejects.toThrow('Output failed') + ).rejects.toBe(failure) + await expect(instance.release()).rejects.toBe(failure) + await Promise.all(updates) expect(instance.abortSignal.aborted).toBe(true) await expect(fileExists(instance.paths.infoFile)).resolves.toBe(false) await expect(fileExists(instance.paths.lockFile)).resolves.toBe(false) + const restarted = await acquireDevInstance({command: 'app dev', directory}) + await restarted.release() }) }) }) diff --git a/packages/cli-kit/src/public/node/dev-instance.ts b/packages/cli-kit/src/public/node/dev-instance.ts index b2be6ab4dfe..a35f67917b4 100644 --- a/packages/cli-kit/src/public/node/dev-instance.ts +++ b/packages/cli-kit/src/public/node/dev-instance.ts @@ -356,8 +356,8 @@ async function createDevInstance(options: AcquireDevInstanceOptions, paths: DevI try { const results = await Promise.allSettled([...outputs].map((output) => endStream(output))) const failure = results.find((result) => result.status === 'rejected') - if (failure?.status === 'rejected') throw failure.reason await updates + if (failure?.status === 'rejected') throw failure.reason } finally { released = true process.removeListener('exit', removeInfoOnExit) From 81032addec0770020a2ee1a08a0e14c52b2104a4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Isaac=20Rold=C3=A1n?= Date: Tue, 20 Oct 2026 18:18:50 +0200 Subject: [PATCH 10/14] Keep distinct dev identities in separate session files --- .../src/public/node/dev-instance.test.ts | 36 ++++++++++++++++++- .../cli-kit/src/public/node/dev-instance.ts | 7 ++-- 2 files changed, 40 insertions(+), 3 deletions(-) diff --git a/packages/cli-kit/src/public/node/dev-instance.test.ts b/packages/cli-kit/src/public/node/dev-instance.test.ts index 416fcb59ba9..5081ecaae5b 100644 --- a/packages/cli-kit/src/public/node/dev-instance.test.ts +++ b/packages/cli-kit/src/public/node/dev-instance.test.ts @@ -104,7 +104,7 @@ describe('acquireDevInstance', () => { // Then const paths = devInstancePaths({command: 'app dev', directory, key: 'shopify.app.toml'}) - expect(paths.infoFile).toMatch(/\.shopify\/dev\/app-dev-shopify\.app\.toml\.json$/) + expect(paths.infoFile).toMatch(/\.shopify\/dev\/app-dev-shopify\.app\.toml-[a-f0-9]{12}\.json$/) const info = JSON.parse(await readFile(paths.infoFile)) expect(info).toMatchObject({ schemaVersion: 1, @@ -150,6 +150,40 @@ describe('acquireDevInstance', () => { }) }) + test.each([ + [ + {command: 'app dev', key: 'shopify.app.Production.toml'}, + {command: 'app dev', key: 'shopify.app.production.toml'}, + ], + [ + {command: 'app dev', key: 'store:one'}, + {command: 'app dev', key: 'store/one'}, + ], + [ + {command: 'app dev-one', key: 'two'}, + {command: 'app dev', key: 'one-two'}, + ], + ])('keeps normalized identities separate: %j and %j', async (firstIdentity, secondIdentity) => { + await inTemporaryDirectory(async (directory) => { + const first = await acquireDevInstance({...firstIdentity, directory}) + try { + const second = await acquireDevInstance({...secondIdentity, directory}) + try { + expect(first.paths.infoFile).not.toBe(second.paths.infoFile) + await first.log({source: 'web', message: 'First session'}) + await second.log({source: 'web', message: 'Second session'}) + expect((await readLog(first.paths.logFile)).map(({message}) => message)).toEqual(['First session']) + expect((await readLog(second.paths.logFile)).map(({message}) => message)).toEqual(['Second session']) + await expect(listDevInstances(directory)).resolves.toHaveLength(2) + } finally { + await second.release() + } + } finally { + await first.release() + } + }) + }) + test('takes over a lock left behind by a process that no longer exists', async () => { await inTemporaryDirectory(async (directory) => { // Given diff --git a/packages/cli-kit/src/public/node/dev-instance.ts b/packages/cli-kit/src/public/node/dev-instance.ts index a35f67917b4..3a2154b8952 100644 --- a/packages/cli-kit/src/public/node/dev-instance.ts +++ b/packages/cli-kit/src/public/node/dev-instance.ts @@ -5,7 +5,7 @@ import {AbortError} from './error.js' import {outputDebug, unstyled} from './output.js' import {defineJsonOutputSchema, formatJsonOutputTimestamp, jsonOutputTimestampSchema} from './json-output-schema.js' import {zod} from './schema.js' -import {randomUUID} from './crypto.js' +import {hashString, randomUUID} from './crypto.js' import { ConcurrentOutputContext, getConcurrentOutputContext, @@ -534,12 +534,15 @@ function isLockedError(error: unknown): boolean { } function devInstanceSlug({command, key}: DevInstanceIdentity): string { - return [command, key] + const slug = [command, key] .filter(Boolean) .join('-') .toLowerCase() .replace(/[^a-z0-9._-]+/g, '-') .replace(/^-+|-+$/g, '') + // Normalization can merge distinct identities, so hash the original command/key pair. + const identityHash = hashString(JSON.stringify([command, key ?? ''])).slice(0, 12) + return `${slug}-${identityHash}` } function devInstanceAlreadyRunningError( From 7b2e0890694ff61769f4c86c37faa3277117ab6b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Isaac=20Rold=C3=A1n?= Date: Tue, 20 Oct 2026 18:30:38 +0200 Subject: [PATCH 11/14] Shorten dev instance filename hashes to four characters --- packages/cli-kit/src/public/node/dev-instance.test.ts | 2 +- packages/cli-kit/src/public/node/dev-instance.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/cli-kit/src/public/node/dev-instance.test.ts b/packages/cli-kit/src/public/node/dev-instance.test.ts index 5081ecaae5b..25aecabd84e 100644 --- a/packages/cli-kit/src/public/node/dev-instance.test.ts +++ b/packages/cli-kit/src/public/node/dev-instance.test.ts @@ -104,7 +104,7 @@ describe('acquireDevInstance', () => { // Then const paths = devInstancePaths({command: 'app dev', directory, key: 'shopify.app.toml'}) - expect(paths.infoFile).toMatch(/\.shopify\/dev\/app-dev-shopify\.app\.toml-[a-f0-9]{12}\.json$/) + expect(paths.infoFile).toMatch(/\.shopify\/dev\/app-dev-shopify\.app\.toml-[a-f0-9]{4}\.json$/) const info = JSON.parse(await readFile(paths.infoFile)) expect(info).toMatchObject({ schemaVersion: 1, diff --git a/packages/cli-kit/src/public/node/dev-instance.ts b/packages/cli-kit/src/public/node/dev-instance.ts index 3a2154b8952..eb4235b780c 100644 --- a/packages/cli-kit/src/public/node/dev-instance.ts +++ b/packages/cli-kit/src/public/node/dev-instance.ts @@ -541,7 +541,7 @@ function devInstanceSlug({command, key}: DevInstanceIdentity): string { .replace(/[^a-z0-9._-]+/g, '-') .replace(/^-+|-+$/g, '') // Normalization can merge distinct identities, so hash the original command/key pair. - const identityHash = hashString(JSON.stringify([command, key ?? ''])).slice(0, 12) + const identityHash = hashString(JSON.stringify([command, key ?? ''])).slice(0, 4) return `${slug}-${identityHash}` } From 48e1e4d0cee84abc1a544157cdc7e750c02de00a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Isaac=20Rold=C3=A1n?= Date: Wed, 21 Oct 2026 13:26:29 +0200 Subject: [PATCH 12/14] Add scoped dev instance lifecycle helper --- .../src/public/node/dev-instance.test.ts | 29 +++++++++++++++++++ .../cli-kit/src/public/node/dev-instance.ts | 19 ++++++++++++ 2 files changed, 48 insertions(+) diff --git a/packages/cli-kit/src/public/node/dev-instance.test.ts b/packages/cli-kit/src/public/node/dev-instance.test.ts index 25aecabd84e..3c1d94f22ee 100644 --- a/packages/cli-kit/src/public/node/dev-instance.test.ts +++ b/packages/cli-kit/src/public/node/dev-instance.test.ts @@ -6,6 +6,7 @@ import { isProcessAlive, listDevInstances, readDevInstance, + withDevInstance, } from './dev-instance.js' import {AbortError} from './error.js' import {fileExists, inTemporaryDirectory, mkdir, readFile, rmdir, writeFile} from './fs.js' @@ -39,6 +40,34 @@ async function readLog(path: string) { .map((line) => JSON.parse(line)) } +describe('withDevInstance', () => { + test.each([false, true])('flushes output and releases ownership when the callback throws: %s', async (throws) => { + await inTemporaryDirectory(async (directory) => { + const identity = {command: 'app dev', directory} + const failure = new Error('Dev failed') + const result = withDevInstance(identity, async (instance) => { + expect((await readDevInstance(identity))?.sessionId).toBe(instance.info.sessionId) + instance.tee(collectingStream().stream, {source: 'web'}).write('Final partial line') + if (throws) throw failure + return 'Dev finished' + }) + if (throws) { + await expect(result).rejects.toBe(failure) + } else { + await expect(result).resolves.toBe('Dev finished') + } + const paths = devInstancePaths(identity) + await expect(readDevInstance(identity)).resolves.toBeUndefined() + await expect(fileExists(paths.lockFile)).resolves.toBe(false) + expect(await readLog(paths.logFile)).toEqual([ + expect.objectContaining({source: 'web', message: 'Final partial line'}), + ]) + const restarted = await acquireDevInstance(identity) + await restarted.release() + }) + }) +}) + describe('acquireDevInstance', () => { test('does not include CLI fatal errors in the session log schema', () => { expect(() => diff --git a/packages/cli-kit/src/public/node/dev-instance.ts b/packages/cli-kit/src/public/node/dev-instance.ts index eb4235b780c..593cd4a4457 100644 --- a/packages/cli-kit/src/public/node/dev-instance.ts +++ b/packages/cli-kit/src/public/node/dev-instance.ts @@ -182,6 +182,25 @@ export async function acquireDevInstance(options: AcquireDevInstanceOptions): Pr } } +/** + * Acquires a dev instance and releases it after the callback completes or throws. + * + * @param options - The command, project and output options for the instance. + * @param execute - The work to run while the instance is owned. + * @returns The callback's result after instance cleanup completes. + */ +export async function withDevInstance( + options: AcquireDevInstanceOptions, + execute: (instance: DevInstance) => T | Promise, +): Promise { + const instance = await acquireDevInstance(options) + try { + return await execute(instance) + } finally { + await instance.release() + } +} + async function createDevInstance(options: AcquireDevInstanceOptions, paths: DevInstancePaths): Promise { const previous = await readDevInstanceInfo(paths.infoFile) if (previous && isProcessAlive(previous.pid)) throw devInstanceAlreadyRunningError(options, previous) From e2477609be41455bcf9212f8ca61714c440d6df3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Isaac=20Rold=C3=A1n?= Date: Wed, 21 Oct 2026 13:27:22 +0200 Subject: [PATCH 13/14] Separate dev instance ownership from output handling --- .../src/private/node/dev-instance-output.ts | 176 ++++++++++++++++++ .../src/private/node/output-context.ts | 18 ++ .../node/ui/components/ConcurrentOutput.tsx | 25 +-- .../src/public/node/dev-instance.test.ts | 2 +- .../cli-kit/src/public/node/dev-instance.ts | 172 ++--------------- 5 files changed, 216 insertions(+), 177 deletions(-) create mode 100644 packages/cli-kit/src/private/node/dev-instance-output.ts create mode 100644 packages/cli-kit/src/private/node/output-context.ts diff --git a/packages/cli-kit/src/private/node/dev-instance-output.ts b/packages/cli-kit/src/private/node/dev-instance-output.ts new file mode 100644 index 00000000000..44f069c60a3 --- /dev/null +++ b/packages/cli-kit/src/private/node/dev-instance-output.ts @@ -0,0 +1,176 @@ +import {ConcurrentOutputContext, getConcurrentOutputContext, useConcurrentOutputContext} from './output-context.js' +import {unstyled} from '../../public/node/output.js' +import { + defineJsonOutputSchema, + formatJsonOutputTimestamp, + jsonOutputTimestampSchema, +} from '../../public/node/json-output-schema.js' +import {zod} from '../../public/node/schema.js' +import {createWriteStream} from 'fs' +import {Writable, type WritableOptions} from 'stream' +import {finished} from 'stream/promises' +import {StringDecoder} from 'string_decoder' +import {once} from 'events' + +export const devInstanceLogOutputSchema = defineJsonOutputSchema({ + name: 'DevInstanceLogEntry', + schema: zod + .object({ + timestamp: jsonOutputTimestampSchema, + source: zod.string().describe('What produced the line, for example a process prefix like web or extensions.'), + level: zod.enum(['debug', 'info', 'warning', 'error']).optional(), + message: zod.string(), + }) + .strict(), +}) + +/** One line of a dev instance's JSON Lines log. */ +export type DevInstanceLogEntry = zod.infer + +interface DevInstanceOutputOptions { + logFile: string + logOutput?: Writable + isActive: () => boolean + onError: (error: unknown) => void +} + +export function createDevInstanceOutput({logFile, logOutput, isActive, onError}: DevInstanceOutputOptions) { + // Each run starts a fresh log. + const logStream = createWriteStream(logFile, {flags: 'w', mode: 0o600}) + logStream.on('error', onError) + logOutput?.on('error', onError) + const opened = once(logStream, 'open') + const outputs = new Set() + + const writeLogEntry = async (entry: Omit & {timestamp?: string}) => { + if (!isActive()) return + const record = devInstanceLogOutputSchema.validate({ + ...entry, + timestamp: entry.timestamp ?? formatJsonOutputTimestamp(new Date()), + }) + const line = `${JSON.stringify(record)}\n` + try { + await writeToStream(logStream, line) + if (logOutput) await writeToStream(logOutput, line) + } catch (error) { + onError(error) + throw error + } + } + + return { + opened, + log: writeLogEntry, + tee: ( + target: Writable, + {source, stringWritesAreMessages = false}: {source: string; stringWritesAreMessages?: boolean}, + ) => { + const decoder = new StringDecoder('utf8') + let pending = '' + let currentSource = source + const flushLines = async (text: string) => { + const lines = text.split(/\r?\n/) + pending = lines.pop() ?? '' + await Promise.all( + lines.map(async (line) => { + const message = unstyled(line).trimEnd() + if (message.length > 0) await writeLogEntry({source: currentSource, message}) + }), + ) + } + const flush = () => flushLines(`${pending}${decoder.end()}\n`) + const output = new DevInstanceOutput({ + decodeStrings: !stringWritesAreMessages, + write(chunk: Buffer | string, encoding, callback) { + const nextSource = getConcurrentOutputContext()?.outputPrefix ?? source + const completeMessage = stringWritesAreMessages && typeof chunk === 'string' + const previous = nextSource !== currentSource || completeMessage ? flush() : Promise.resolve() + currentSource = nextSource + Promise.all([ + previous, + writeToStream(target, chunk, encoding), + flushLines( + pending + + decoder.write(typeof chunk === 'string' ? Buffer.from(chunk, encoding) : chunk) + + (completeMessage ? '\n' : ''), + ), + ]) + .then(() => callback()) + .catch(callback) + }, + final(callback) { + flush() + .then(() => callback()) + .catch(callback) + }, + }) + const onTargetError = (error: Error) => output.destroy(error) + target.on('error', onTargetError) + output.on('error', (error) => onError(error)) + output.once('close', () => { + target.off('error', onTargetError) + }) + outputs.add(output) + return output + }, + flush: () => Promise.allSettled([...outputs].map((output) => endStream(output))), + close: async () => { + try { + await endStream(logStream) + if (logOutput && !logOutput.destroyed) await writeToStream(logOutput, '') + } finally { + logOutput?.off('error', onError) + } + }, + } +} + +// Writable processes queued chunks in an earlier write's context. Capture context when each chunk is submitted. +class DevInstanceOutput extends Writable { + private readonly contexts: (ConcurrentOutputContext | undefined)[] = [] + + constructor(options: WritableOptions & {write: NonNullable}) { + super({ + ...options, + write: (chunk, encoding, callback) => { + useConcurrentOutputContext(this.contexts.shift() ?? {}, () => + options.write.call(this, chunk, encoding, callback), + ) + }, + }) + } + + write( + chunk: string | Uint8Array, + encoding?: BufferEncoding | ((error?: Error | null) => void), + callback?: (error?: Error | null) => void, + ): boolean { + this.contexts.push(getConcurrentOutputContext()) + return typeof encoding === 'string' + ? super.write(chunk, encoding, callback) + : super.write(chunk, encoding ?? callback) + } + + end( + chunk?: string | Uint8Array | null | (() => void), + encoding?: BufferEncoding | (() => void), + callback?: () => void, + ): this { + if (chunk !== undefined && chunk !== null && typeof chunk !== 'function') + this.contexts.push(getConcurrentOutputContext()) + if (typeof chunk === 'function') return super.end(chunk) + return typeof encoding === 'string' ? super.end(chunk, encoding, callback) : super.end(chunk, encoding ?? callback) + } +} + +async function writeToStream(stream: Writable, chunk: Buffer | string, encoding: BufferEncoding = 'utf8') { + await new Promise((resolve, reject) => { + stream.write(chunk, encoding, (error) => (error ? reject(error) : resolve())) + }) +} + +async function endStream(stream: Writable) { + const completed = finished(stream, {cleanup: true}) + stream.end() + await completed +} diff --git a/packages/cli-kit/src/private/node/output-context.ts b/packages/cli-kit/src/private/node/output-context.ts new file mode 100644 index 00000000000..0c418c572e2 --- /dev/null +++ b/packages/cli-kit/src/private/node/output-context.ts @@ -0,0 +1,18 @@ +import {AsyncLocalStorage} from 'node:async_hooks' + +export interface ConcurrentOutputContext { + outputPrefix?: string + /** Controls ANSI stripping for terminal output. JSON diagnostics are always unstyled. */ + stripAnsi?: boolean +} + +const outputContextStore = new AsyncLocalStorage() + +export function useConcurrentOutputContext(context: ConcurrentOutputContext, callback: () => T): T { + return outputContextStore.run(context, callback) +} + +/** Returns the source and styling context for the current output write. */ +export function getConcurrentOutputContext(): ConcurrentOutputContext | undefined { + return outputContextStore.getStore() +} diff --git a/packages/cli-kit/src/private/node/ui/components/ConcurrentOutput.tsx b/packages/cli-kit/src/private/node/ui/components/ConcurrentOutput.tsx index 6e5e59c0f68..6ea287768fa 100644 --- a/packages/cli-kit/src/private/node/ui/components/ConcurrentOutput.tsx +++ b/packages/cli-kit/src/private/node/ui/components/ConcurrentOutput.tsx @@ -1,3 +1,4 @@ +import {ConcurrentOutputContext, getConcurrentOutputContext, useConcurrentOutputContext} from '../../output-context.js' import {outputInfo, type OutputProcess} from '../../../../public/node/output.js' import {AbortSignal} from '../../../../public/node/abort.js' import {useComplete} from '../../ui.js' @@ -7,7 +8,6 @@ import figures from 'figures' import stripAnsi from 'strip-ansi' import {Writable} from 'stream' -import {AsyncLocalStorage} from 'node:async_hooks' import {StringDecoder} from 'node:string_decoder' export interface ConcurrentOutputProps { @@ -45,23 +45,6 @@ function currentTime() { return `${hours}:${minutes}:${seconds}` } -interface ConcurrentOutputContext { - outputPrefix?: string - /** Controls ANSI stripping for terminal output. JSON diagnostics are always unstyled. */ - stripAnsi?: boolean -} - -const outputContextStore = new AsyncLocalStorage() - -function useConcurrentOutputContext(context: ConcurrentOutputContext, callback: () => T): T { - return outputContextStore.run(context, callback) -} - -/** Returns the source and styling context for the current output write. */ -export function getConcurrentOutputContext(): ConcurrentOutputContext | undefined { - return outputContextStore.getStore() -} - /** Runs finite processes concurrently and routes their output through the shared diagnostic context. */ export async function runConcurrentProcessesForJson({ processes, @@ -84,7 +67,7 @@ export async function runConcurrentProcessesForJson({ next() return } - const currentPrefix = outputContextStore.getStore()?.outputPrefix ?? process.prefix + const currentPrefix = getConcurrentOutputContext()?.outputPrefix ?? process.prefix if (!hasPendingLine) prefix = currentPrefix const lines = (pending + decoder.write(chunk)).split(/\r?\n/) pending = lines.pop() ?? '' @@ -202,7 +185,7 @@ const ConcurrentOutput: FunctionComponent = ({ (process: OutputProcess, prefixes: string[]) => { return new Writable({ write(chunk, _encoding, next) { - const context = outputContextStore.getStore() + const context = getConcurrentOutputContext() const prefix = context?.outputPrefix ?? process.prefix const shouldStripAnsi = context?.stripAnsi ?? true const log = chunk.toString('utf8').replace(/(\n)$/, '') @@ -296,4 +279,4 @@ const ConcurrentOutput: FunctionComponent = ({ ) } -export {ConcurrentOutput, ConcurrentOutputContext, useConcurrentOutputContext} +export {ConcurrentOutput, ConcurrentOutputContext, getConcurrentOutputContext, useConcurrentOutputContext} diff --git a/packages/cli-kit/src/public/node/dev-instance.test.ts b/packages/cli-kit/src/public/node/dev-instance.test.ts index 3c1d94f22ee..066890ab9c5 100644 --- a/packages/cli-kit/src/public/node/dev-instance.test.ts +++ b/packages/cli-kit/src/public/node/dev-instance.test.ts @@ -59,7 +59,7 @@ describe('withDevInstance', () => { const paths = devInstancePaths(identity) await expect(readDevInstance(identity)).resolves.toBeUndefined() await expect(fileExists(paths.lockFile)).resolves.toBe(false) - expect(await readLog(paths.logFile)).toEqual([ + await expect(readLog(paths.logFile)).resolves.toEqual([ expect.objectContaining({source: 'web', message: 'Final partial line'}), ]) const restarted = await acquireDevInstance(identity) diff --git a/packages/cli-kit/src/public/node/dev-instance.ts b/packages/cli-kit/src/public/node/dev-instance.ts index 593cd4a4457..b1bbca1663c 100644 --- a/packages/cli-kit/src/public/node/dev-instance.ts +++ b/packages/cli-kit/src/public/node/dev-instance.ts @@ -2,22 +2,14 @@ import {getOrCreateHiddenShopifyFolder} from './hidden-folder.js' import {joinPath} from './path.js' import {fileExists, glob, mkdir, readFile, removeFile, removeFileSync, renameFile, rmdir} from './fs.js' import {AbortError} from './error.js' -import {outputDebug, unstyled} from './output.js' -import {defineJsonOutputSchema, formatJsonOutputTimestamp, jsonOutputTimestampSchema} from './json-output-schema.js' -import {zod} from './schema.js' +import {outputDebug} from './output.js' +import {formatJsonOutputTimestamp} from './json-output-schema.js' import {hashString, randomUUID} from './crypto.js' -import { - ConcurrentOutputContext, - getConcurrentOutputContext, - useConcurrentOutputContext, -} from '../../private/node/ui/components/ConcurrentOutput.js' +import {createDevInstanceOutput, type DevInstanceLogEntry} from '../../private/node/dev-instance-output.js' import lockfile from 'proper-lockfile' -import fs, {createWriteStream, readFileSync, type WriteStream} from 'fs' +import fs, {readFileSync} from 'fs' import {mkdir as mkdirExclusive, rmdir as removeEmptyDirectory, writeFile as writeFileWithMode} from 'fs/promises' -import {Writable, type WritableOptions} from 'stream' -import {finished} from 'stream/promises' -import {StringDecoder} from 'string_decoder' -import {once} from 'events' +import {Writable} from 'stream' /** * A stable error code for JSON consumers when a long-running command refuses to start because another instance of @@ -52,20 +44,7 @@ export interface DevInstanceInfo { details: {[key: string]: unknown} } -export const devInstanceLogOutputSchema = defineJsonOutputSchema({ - name: 'DevInstanceLogEntry', - schema: zod - .object({ - timestamp: jsonOutputTimestampSchema, - source: zod.string().describe('What produced the line, for example a process prefix like web or extensions.'), - level: zod.enum(['debug', 'info', 'warning', 'error']).optional(), - message: zod.string(), - }) - .strict(), -}) - -/** One line of a dev instance's JSON Lines log. */ -export type DevInstanceLogEntry = zod.infer +export {devInstanceLogOutputSchema, type DevInstanceLogEntry} from '../../private/node/dev-instance-output.js' export interface DevInstancePaths { /** The folder that holds every dev instance of the project: `.shopify/dev`. */ @@ -269,15 +248,14 @@ async function createDevInstance(options: AcquireDevInstanceOptions, paths: DevI details: {...(options.details ?? {})}, } - // Each run starts a fresh log. - const logStream: WriteStream = createWriteStream(paths.logFile, {flags: 'w', mode: 0o600}) - const onOutputError = (error: Error) => abortController.abort(error) - logStream.on('error', onOutputError) - options.logOutput?.on('error', onOutputError) - let updates = Promise.resolve() - const outputs = new Set() let released = false + const output = createDevInstanceOutput({ + logFile: paths.logFile, + logOutput: options.logOutput, + isActive: () => !released && !compromised, + onError: (error) => abortController.abort(error), + }) let releasePromise: Promise | undefined const removeInfoOnExit = () => { // Remove the lock before the live owner's record, so a new starter cannot enter during cleanup. @@ -288,22 +266,6 @@ async function createDevInstance(options: AcquireDevInstanceOptions, paths: DevI } process.once('exit', removeInfoOnExit) - const writeLogEntry: DevInstance['log'] = async (entry) => { - if (released || compromised) return - const record = devInstanceLogOutputSchema.validate({ - ...entry, - timestamp: entry.timestamp ?? formatJsonOutputTimestamp(new Date()), - }) - const line = `${JSON.stringify(record)}\n` - try { - await writeToStream(logStream, line) - if (options.logOutput) await writeToStream(options.logOutput, line) - } catch (error) { - abortController.abort(error) - throw error - } - } - const instance: DevInstance = { info, paths, @@ -320,60 +282,12 @@ async function createDevInstance(options: AcquireDevInstanceOptions, paths: DevI updates = write.catch(() => {}) await write }, - log: writeLogEntry, - tee: (target, {source, stringWritesAreMessages = false}) => { - const decoder = new StringDecoder('utf8') - let pending = '' - let currentSource = source - const flushLines = async (text: string) => { - const lines = text.split(/\r?\n/) - pending = lines.pop() ?? '' - await Promise.all( - lines.map(async (line) => { - const message = unstyled(line).trimEnd() - if (message.length > 0) await writeLogEntry({source: currentSource, message}) - }), - ) - } - const flush = () => flushLines(`${pending}${decoder.end()}\n`) - const output = new DevInstanceOutput({ - decodeStrings: !stringWritesAreMessages, - write(chunk: Buffer | string, encoding, callback) { - const nextSource = getConcurrentOutputContext()?.outputPrefix ?? source - const completeMessage = stringWritesAreMessages && typeof chunk === 'string' - const previous = nextSource !== currentSource || completeMessage ? flush() : Promise.resolve() - currentSource = nextSource - Promise.all([ - previous, - writeToStream(target, chunk, encoding), - flushLines( - pending + - decoder.write(typeof chunk === 'string' ? Buffer.from(chunk, encoding) : chunk) + - (completeMessage ? '\n' : ''), - ), - ]) - .then(() => callback()) - .catch(callback) - }, - final(callback) { - flush() - .then(() => callback()) - .catch(callback) - }, - }) - const onTargetError = (error: Error) => output.destroy(error) - target.on('error', onTargetError) - output.on('error', (error) => abortController.abort(error)) - output.once('close', () => { - target.off('error', onTargetError) - }) - outputs.add(output) - return output - }, + log: output.log, + tee: output.tee, release: () => { releasePromise ??= (async () => { try { - const results = await Promise.allSettled([...outputs].map((output) => endStream(output))) + const results = await output.flush() const failure = results.find((result) => result.status === 'rejected') await updates if (failure?.status === 'rejected') throw failure.reason @@ -381,10 +295,8 @@ async function createDevInstance(options: AcquireDevInstanceOptions, paths: DevI released = true process.removeListener('exit', removeInfoOnExit) try { - await endStream(logStream) - if (options.logOutput && !options.logOutput.destroyed) await writeToStream(options.logOutput, '') + await output.close() } finally { - options.logOutput?.off('error', onOutputError) const owned = ownsInfo() await releaseLock().catch((error: Error) => outputDebug(`Failed to release the dev instance lock: ${error.message}`), @@ -398,7 +310,7 @@ async function createDevInstance(options: AcquireDevInstanceOptions, paths: DevI } try { - await once(logStream, 'open') + await output.opened await writeDevInstanceInfo(paths.infoFile, info) published = true return instance @@ -410,56 +322,6 @@ async function createDevInstance(options: AcquireDevInstanceOptions, paths: DevI } } -// Writable processes queued chunks in an earlier write's context. Capture context when each chunk is submitted. -class DevInstanceOutput extends Writable { - private readonly contexts: (ConcurrentOutputContext | undefined)[] = [] - - constructor(options: WritableOptions & {write: NonNullable}) { - super({ - ...options, - write: (chunk, encoding, callback) => { - useConcurrentOutputContext(this.contexts.shift() ?? {}, () => - options.write.call(this, chunk, encoding, callback), - ) - }, - }) - } - - write( - chunk: string | Uint8Array, - encoding?: BufferEncoding | ((error?: Error | null) => void), - callback?: (error?: Error | null) => void, - ): boolean { - this.contexts.push(getConcurrentOutputContext()) - return typeof encoding === 'string' - ? super.write(chunk, encoding, callback) - : super.write(chunk, encoding ?? callback) - } - - end( - chunk?: string | Uint8Array | null | (() => void), - encoding?: BufferEncoding | (() => void), - callback?: () => void, - ): this { - if (chunk !== undefined && chunk !== null && typeof chunk !== 'function') - this.contexts.push(getConcurrentOutputContext()) - if (typeof chunk === 'function') return super.end(chunk) - return typeof encoding === 'string' ? super.end(chunk, encoding, callback) : super.end(chunk, encoding ?? callback) - } -} - -async function writeToStream(stream: Writable, chunk: Buffer | string, encoding: BufferEncoding = 'utf8') { - await new Promise((resolve, reject) => { - stream.write(chunk, encoding, (error) => (error ? reject(error) : resolve())) - }) -} - -async function endStream(stream: Writable) { - const completed = finished(stream, {cleanup: true}) - stream.end() - await completed -} - /** * Returns the running instance of a command for a project, if there is one. * From c8369814e9f4355a5e19ccf5bdbd6c22936d558a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Isaac=20Rold=C3=A1n?= Date: Wed, 21 Oct 2026 15:13:30 +0200 Subject: [PATCH 14/14] Delete .changeset/cli-kit-dev-instance.md --- .changeset/cli-kit-dev-instance.md | 5 ----- 1 file changed, 5 deletions(-) delete mode 100644 .changeset/cli-kit-dev-instance.md diff --git a/.changeset/cli-kit-dev-instance.md b/.changeset/cli-kit-dev-instance.md deleted file mode 100644 index 6a048e49412..00000000000 --- a/.changeset/cli-kit-dev-instance.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -'@shopify/cli-kit': minor ---- - -Add dev session files for ownership, status, and JSONL logs.