From e1b8b48d035a0ceb1d0858285356daa5c1405eeb Mon Sep 17 00:00:00 2001 From: Arul Sharma <31745423+arul28@users.noreply.github.com> Date: Wed, 29 Apr 2026 22:52:16 -0400 Subject: [PATCH 1/4] ship: checkpoint before automate/finalize --- apps/ade-cli/src/bootstrap.ts | 29 + apps/ade-cli/src/cli.test.ts | 127 ++ apps/ade-cli/src/cli.ts | 411 +++- apps/desktop/scripts/dev.cjs | 28 +- apps/desktop/src/main/main.ts | 36 +- .../src/main/services/adeActions/registry.ts | 35 + .../appControl/appControlService.test.ts | 215 ++ .../services/appControl/appControlService.ts | 2026 +++++++++++++++++ .../main/services/chat/agentChatService.ts | 12 +- .../src/main/services/ipc/registerIpc.ts | 507 +++++ .../src/main/services/pty/ptyService.ts | 232 +- .../main/services/sessions/sessionService.ts | 27 +- apps/desktop/src/main/services/state/kvDb.ts | 3 + apps/desktop/src/preload/global.d.ts | 53 + apps/desktop/src/preload/preload.ts | 79 + .../components/chat/AgentChatComposer.tsx | 340 ++- .../chat/AgentChatPane.submit.test.tsx | 7 + .../components/chat/AgentChatPane.tsx | 593 ++++- .../components/chat/ChatAppControlPanel.tsx | 1419 ++++++++++++ .../components/chat/ChatTerminalDrawer.tsx | 339 ++- .../components/terminals/SessionCard.tsx | 28 +- .../components/terminals/SessionListPane.tsx | 174 +- .../components/terminals/WorkStartSurface.tsx | 2 +- apps/desktop/src/shared/adeCliGuidance.ts | 4 +- apps/desktop/src/shared/ipc.ts | 21 + apps/desktop/src/shared/types/appControl.ts | 203 ++ apps/desktop/src/shared/types/index.ts | 1 + apps/desktop/src/shared/types/sessions.ts | 59 +- apps/ios/ADE/Models/RemoteModels.swift | 3 + apps/ios/ADE/Resources/DatabaseBootstrap.sql | 5 + apps/ios/ADE/Services/Database.swift | 26 +- apps/ios/ADETests/ADETests.swift | 60 + docs/ARCHITECTURE.md | 20 +- docs/features/chat/README.md | 4 +- docs/features/chat/composer-and-ui.md | 1 + docs/features/computer-use/README.md | 10 +- docs/features/computer-use/app-control.md | 152 ++ .../features/terminals-and-sessions/README.md | 44 +- 38 files changed, 7081 insertions(+), 254 deletions(-) create mode 100644 apps/desktop/src/main/services/appControl/appControlService.test.ts create mode 100644 apps/desktop/src/main/services/appControl/appControlService.ts create mode 100644 apps/desktop/src/renderer/components/chat/ChatAppControlPanel.tsx create mode 100644 apps/desktop/src/shared/types/appControl.ts create mode 100644 docs/features/computer-use/app-control.md diff --git a/apps/ade-cli/src/bootstrap.ts b/apps/ade-cli/src/bootstrap.ts index a62535647..305074b52 100644 --- a/apps/ade-cli/src/bootstrap.ts +++ b/apps/ade-cli/src/bootstrap.ts @@ -73,6 +73,10 @@ import { createIosSimulatorService, type IosSimulatorService, } from "../../desktop/src/main/services/ios/iosSimulatorService"; +import { + createAppControlService, + type AppControlService, +} from "../../desktop/src/main/services/appControl/appControlService"; import type { createFileService } from "../../desktop/src/main/services/files/fileService"; import { createAutomationService, @@ -168,6 +172,7 @@ export type AdeRuntime = { automationPlannerService?: ReturnType | null; computerUseArtifactBrokerService: ComputerUseArtifactBrokerService; iosSimulatorService?: IosSimulatorService | null; + appControlService?: AppControlService | null; orchestratorService: ReturnType; aiOrchestratorService: ReturnType; missionBudgetService?: ReturnType | null; @@ -471,6 +476,28 @@ export async function createAdeRuntime(args: { projectRoot: string; workspaceRoo projectRoot, logger, }); + const appControlService = createAppControlService({ + projectRoot, + logger, + ptyService, + resolveLaneId: async ({ cwd, projectRoot: requestedProjectRoot, laneId, chatSessionId }) => { + const explicitLaneId = laneId?.trim(); + if (explicitLaneId) return explicitLaneId; + const chatId = chatSessionId?.trim(); + if (chatId) { + const chatSession = sessionService.get(chatId); + if (chatSession?.laneId) return chatSession.laneId; + } + const targetRoot = path.resolve(cwd || requestedProjectRoot || projectRoot); + const lanes = await laneService.list({ includeArchived: false }); + const matchingLane = lanes.find((lane) => { + const worktreePath = path.resolve(lane.worktreePath); + const attachedRootPath = lane.attachedRootPath ? path.resolve(lane.attachedRootPath) : null; + return targetRoot === worktreePath || targetRoot.startsWith(`${worktreePath}${path.sep}`) || targetRoot === attachedRootPath; + }); + return matchingLane?.id ?? lanes[0]?.id ?? null; + }, + }); const aiOrchestratorService = createAiOrchestratorService({ db, @@ -573,6 +600,7 @@ export async function createAdeRuntime(args: { projectRoot: string; workspaceRoo automationPlannerService, computerUseArtifactBrokerService, iosSimulatorService, + appControlService, orchestratorService, aiOrchestratorService, eventBuffer, @@ -581,6 +609,7 @@ export async function createAdeRuntime(args: { projectRoot: string; workspaceRoo swallow(() => automationService.dispose()); swallow(() => processService.disposeAll()); swallow(() => iosSimulatorService.dispose()); + swallow(() => appControlService.dispose()); swallow(() => headlessLinearServices.dispose()); swallow(() => aiOrchestratorService.dispose()); swallow(() => testService.disposeAll()); diff --git a/apps/ade-cli/src/cli.test.ts b/apps/ade-cli/src/cli.test.ts index d02e9fb86..c6e0614ee 100644 --- a/apps/ade-cli/src/cli.test.ts +++ b/apps/ade-cli/src/cli.test.ts @@ -1311,6 +1311,133 @@ describe("ADE CLI", () => { } }); + it("app-control status maps to app_control actions", () => { + const status = buildCliPlan(["app-control", "status"]); + expect(status.kind).toBe("execute"); + if (status.kind !== "execute") return; + expect(status.steps[0]?.params).toMatchObject({ + name: "run_ade_action", + arguments: { domain: "app_control", action: "getStatus" }, + }); + }); + + it("app-control launch requires a command and supports aliases", () => { + const launch = buildCliPlan(["app-control", "launch", "--command", "npm run dev", "--debug-port", "9333", "--force"]); + expect(launch.kind).toBe("execute"); + if (launch.kind !== "execute") return; + expect(launch.steps[0]?.params).toMatchObject({ + arguments: { + domain: "app_control", + action: "launch", + args: { + command: "npm run dev", + debugPort: 9333, + force: true, + }, + }, + }); + + const command = buildCliPlan(["electron", "launch", "pnpm", "dev"]); + expect(command.kind).toBe("execute"); + if (command.kind !== "execute") return; + expect(command.steps[0]?.params).toMatchObject({ + arguments: { + domain: "app_control", + action: "launch", + args: { command: "pnpm dev" }, + }, + }); + }); + + it("terminal read and write map to terminal actions", () => { + const read = buildCliPlan(["terminal", "read", "--chat-session", "chat-1", "--max-bytes", "500"]); + expect(read.kind).toBe("execute"); + if (read.kind !== "execute") return; + expect(read.steps[0]?.params).toMatchObject({ + arguments: { + domain: "terminal", + action: "read", + args: { chatSessionId: "chat-1", maxBytes: 500 }, + }, + }); + + const write = buildCliPlan(["terminal", "write", "--terminal", "term-1", "--data", "y\n"]); + expect(write.kind).toBe("execute"); + if (write.kind !== "execute") return; + expect(write.steps[0]?.params).toMatchObject({ + arguments: { + domain: "terminal", + action: "write", + args: { terminalId: "term-1", data: "y\n" }, + }, + }); + }); + + it("app-control logs and terminal write use the active App Control terminal", () => { + const logs = buildCliPlan(["app-control", "logs", "--max-bytes", "1024"]); + expect(logs.kind).toBe("execute"); + if (logs.kind !== "execute") return; + expect(logs.steps[0]?.params).toMatchObject({ + arguments: { + domain: "app_control", + action: "readTerminal", + args: { maxBytes: 1024 }, + }, + }); + + const write = buildCliPlan(["app-control", "terminal", "write", "--data", "y\n"]); + expect(write.kind).toBe("execute"); + if (write.kind !== "execute") return; + expect(write.steps[0]?.params).toMatchObject({ + arguments: { + domain: "app_control", + action: "writeTerminal", + args: { data: "y\n" }, + }, + }); + }); + + it("app-control connect, select, click, and type map to App Control actions", () => { + const connect = buildCliPlan(["app-control", "connect", "--cdp-port", "9222", "--force"]); + expect(connect.kind).toBe("execute"); + if (connect.kind !== "execute") return; + expect(connect.steps[0]?.params).toMatchObject({ + arguments: { + domain: "app_control", + action: "connect", + args: { cdpPort: 9222, force: true }, + }, + }); + + const positionalConnect = buildCliPlan(["app-control", "connect", "9333"]); + expect(positionalConnect.kind).toBe("execute"); + if (positionalConnect.kind !== "execute") return; + expect(positionalConnect.steps[0]?.params).toMatchObject({ + arguments: { domain: "app_control", action: "connect", args: { cdpPort: 9333 } }, + }); + + const select = buildCliPlan(["app-control", "select", "--x", "120", "--y", "420"]); + expect(select.kind).toBe("execute"); + if (select.kind !== "execute") return; + expect(select.steps[0]?.params).toMatchObject({ + arguments: { domain: "app_control", action: "selectPoint", args: { x: 120, y: 420 } }, + }); + + const click = buildCliPlan(["app", "click", "120", "420"]); + expect(click.kind).toBe("execute"); + if (click.kind !== "execute") return; + expect(click.steps[0]?.params).toMatchObject({ + arguments: { domain: "app_control", action: "click", args: { x: 120, y: 420 } }, + }); + + const type = buildCliPlan(["app-control", "type", "--value", "hello", "--text"]); + expect(type.kind).toBe("execute"); + if (type.kind !== "execute") return; + expect(type.steps[0]?.params).toMatchObject({ + arguments: { domain: "app_control", action: "typeText", args: { text: "hello" } }, + }); + }); + it("attaches a rendered lane graph when the plan has the lanes visualizer", () => { const connection = { mode: "headless" as const, diff --git a/apps/ade-cli/src/cli.ts b/apps/ade-cli/src/cli.ts index 36e8e568b..96490da2e 100644 --- a/apps/ade-cli/src/cli.ts +++ b/apps/ade-cli/src/cli.ts @@ -61,6 +61,11 @@ type FormatterId = | "ios-sim-snapshot" | "ios-sim-selection" | "ios-sim-preview" + | "app-control-status" + | "app-control-snapshot" + | "app-control-selection" + | "terminal-list" + | "terminal-read" | "actions-list" | "action-result" | "automation-run-detail"; @@ -265,6 +270,7 @@ const TOP_LEVEL_HELP = `${ADE_BANNER} $ ade prs list | create | path-to-merge Manage PRs, queues, and Path to Merge repair rounds $ ade run defs | ps | start | logs Manage Run tab process definitions and runtime $ ade shell start | write | resize | close Launch and control tracked shell sessions + $ ade terminal list | read | write | signal Control the active in-chat terminal $ ade chat list | create | send | interrupt Work with ADE agent chats $ ade agent spawn --lane --prompt Launch an agent session in ADE $ ade cto state | chats Operate CTO state and Work chats @@ -274,6 +280,7 @@ const TOP_LEVEL_HELP = `${ADE_BANNER} $ ade tests list | run | stop | runs | logs Run configured test suites $ ade proof status | list | screenshot | record Manage proof and computer-use artifacts $ ade ios-sim devices | apps | launch | tap Control iOS Simulator apps, capture, and input + $ ade app-control launch | snapshot | click Inspect and drive Electron apps $ ade memory add | search | pin Use ADE memory $ ade settings action Call project config actions $ ade actions list | run | status Escape hatch for every ADE service action @@ -299,6 +306,8 @@ const TOP_LEVEL_HELP = `${ADE_BANNER} $ ade proof record --seconds 20 $ ade ios-sim apps --text $ ade ios-sim launch --target --text + $ ade app-control launch --command "pnpm dev" --text + $ ade terminal read --chat-session --text Generic ADE action JSON contract: Object-shaped call: @@ -725,6 +734,19 @@ const HELP_BY_COMMAND: Record = { $ ade shell write --data "q" Write data to a PTY $ ade shell resize --cols 120 --rows 36 $ ade shell close Dispose a PTY +`, + terminal: `${ADE_BANNER} + Chat terminal + + Terminal commands control the active in-chat terminal for an ADE chat. Use + desktop socket mode when you want the same terminal the user sees in the app. + + $ ade terminal list --chat-session --text List terminals for a chat + $ ade terminal active --chat-session --text Show the active chat terminal + $ ade terminal read --terminal --text Read terminal scrollback + $ ade app-control logs --text Read the active App Control launch terminal + $ ade terminal write --terminal --data "y\\n" + $ ade terminal signal --terminal --signal SIGINT `, files: `${ADE_BANNER} Files @@ -828,6 +850,52 @@ const HELP_BY_COMMAND: Record = { $ ade ios-sim drag 120 700 120 250 Drag through idb (drag) $ ade ios-sim swipe 120 700 120 250 Swipe through idb (swipe) $ ade ios-sim type "hello" --text Type into the launched app (typeText) +`, + "app-control": `${ADE_BANNER} + App Control + + App Control is ADE's bridge for developer-owned app sessions. The first + supported kind is Electron: ADE can launch or connect to an Electron renderer + that exposes a Chrome DevTools Protocol port, then capture screenshots, DOM + elements, selected UI context, and basic input in the same style as the iOS + simulator drawer. App Control is intentionally a bridge: Playwright, + agent-browser, Computer Use, and other tools may also attach to the same app; + ADE keeps the launch/session state and turns snapshots into chat context. + + Launching runs the command in the chat terminal instead of a hidden child + process. ADE sets ADE_APP_CONTROL_CDP_PORT and ADE_APP_CONTROL_DEBUG_FLAGS in + the environment and auto-forwards debug flags for common npm/pnpm/yarn/bun + script launches and direct electron commands. Custom launchers should forward + ADE_APP_CONTROL_DEBUG_FLAGS or ADE_APP_CONTROL_CDP_PORT. You can also put + {ADE_APP_CONTROL_DEBUG_FLAGS} in the command string for explicit substitution. + + Reuse a Run-tab command: list configured processes with + \`ade settings get --text\`, then pass \`--cwd\` so the launch runs from the + same directory the Run tab uses. Relative cwds resolve against the lane root. + + Discovery and lifecycle: + $ ade app-control status --text Show active session and provider readiness + $ ade app-control launch --command "npm run dev" --text + $ ade app-control launch pnpm dev --text Launch via the visible chat terminal + $ ade app-control launch --command "pnpm dev" --cwd apps/desktop --text + $ ade app-control launch --command "/path/script.sh {ADE_APP_CONTROL_DEBUG_FLAGS}" + $ ade app-control connect --cdp-port 9222 Attach to an already-running app + $ ade app-control logs --text Read the active App Control launch terminal + $ ade app-control terminal write --data "y\\n" Answer a prompt in that terminal + $ ade app-control stop --text Signal the App Control terminal session + $ ade app-control actions --text List every callable app_control action + $ ade terminal read --terminal --text Read a specific chat terminal + $ ade terminal write --chat-session --data "y\\n" Answer a prompt + + Capture and context: + $ ade app-control screenshot --text Capture the active renderer screenshot + $ ade app-control snapshot --text Screenshot + DOM element refs + $ ade app-control inspect --x 120 --y 420 Hit-test a point without committing context + $ ade app-control select --x 120 --y 420 Add selected app context to the drawer chat + + Input: + $ ade app-control click 120 420 Click screenshot coordinates + $ ade app-control type "hello" --text Type text into the focused element `, tests: `${ADE_BANNER} Tests @@ -1087,6 +1155,18 @@ function buildIosSimulatorHelp(args: string[]): string { return HELP_BY_COMMAND["ios-sim"]; } +function buildAppControlHelp(args: string[]): string { + const rawSubcommand = peekFirstPositional(args)?.toLowerCase() ?? ""; + if (!rawSubcommand) return HELP_BY_COMMAND["app-control"]; + const focused = `${HELP_BY_COMMAND["app-control"]} + Focused help for '${rawSubcommand}': + Most subcommands accept --input-json and --arg/--arg-json as escape hatches. + Use "ade app-control actions --text" to inspect the exact service methods. + Use --socket when you want the desktop drawer and CLI to share the same live App Control session. +`; + return focused; +} + function collectGenericObjectArgs(args: string[], base: JsonObject = {}): JsonObject { const input: JsonObject = { ...base }; while (true) { @@ -1928,6 +2008,62 @@ function buildShellPlan(args: string[]): CliPlan { return { kind: "execute", label: `shell ${sub}`, steps: [actionStep("result", "pty", sub, collectGenericObjectArgs(args))] }; } +function buildTerminalPlan(args: string[]): CliPlan { + const sub = firstPositional(args) ?? "active"; + if (sub === "actions") return { kind: "execute", label: "terminal actions", steps: [listActionsStep("actions", "terminal")] }; + const chatSessionId = () => readValue(args, ["--chat-session", "--chat-session-id", "--session", "--session-id"]) ?? process.env.ADE_CHAT_SESSION_ID ?? null; + if (sub === "list" || sub === "ls") { + return { kind: "execute", label: "terminal list", steps: [actionStep("result", "terminal", "list", collectGenericObjectArgs(args, { + chatSessionId: chatSessionId(), + laneId: readValue(args, ["--lane", "--lane-id"]), + limit: readIntOption(args, ["--limit"], undefined), + }))] }; + } + if (sub === "active" || sub === "current") { + return { kind: "execute", label: "terminal active", steps: [actionStep("result", "terminal", "activeForChat", collectGenericObjectArgs(args, { + chatSessionId: requireValue(chatSessionId(), "chatSessionId"), + }))] }; + } + if (sub === "read" || sub === "tail" || sub === "scrollback") { + const terminal = readValue(args, ["--terminal", "--terminal-id"]); + const chat = chatSessionId(); + const maxBytes = readIntOption(args, ["--max-bytes"], undefined); + const since = readIntOption(args, ["--since"], undefined); + return { kind: "execute", label: "terminal read", steps: [actionStep("result", "terminal", "read", collectGenericObjectArgs(args, { + terminalId: terminal ?? firstPositional(args), + chatSessionId: chat, + maxBytes, + since, + }))] }; + } + if (sub === "write" || sub === "send" || sub === "input") { + const terminal = readValue(args, ["--terminal", "--terminal-id"]); + const ptyId = readValue(args, ["--pty", "--pty-id"]); + const chat = chatSessionId(); + const data = readValue(args, ["--data", "--value", "--text"]) ?? args.join(" "); + if (!data.length) throw new CliUsageError("data is required."); + return { kind: "execute", label: "terminal write", steps: [actionStep("result", "terminal", "write", collectGenericObjectArgs(args, { + terminalId: terminal ?? firstPositional(args), + ptyId, + chatSessionId: chat, + data, + }))] }; + } + if (sub === "signal" || sub === "interrupt" || sub === "stop") { + const terminal = readValue(args, ["--terminal", "--terminal-id"]); + const ptyId = readValue(args, ["--pty", "--pty-id"]); + const chat = chatSessionId(); + const signal = readValue(args, ["--signal"]) ?? (sub === "stop" ? "SIGTERM" : "SIGINT"); + return { kind: "execute", label: "terminal signal", steps: [actionStep("result", "terminal", "signal", collectGenericObjectArgs(args, { + terminalId: terminal ?? firstPositional(args), + ptyId, + chatSessionId: chat, + signal, + }))] }; + } + return { kind: "execute", label: `terminal ${sub}`, steps: [actionStep("result", "terminal", sub, collectGenericObjectArgs(args))] }; +} + function buildChatPlan(args: string[]): CliPlan { const sub = firstPositional(args) ?? "list"; if (sub === "actions") return { kind: "execute", label: "chat actions", steps: [listActionsStep("actions", "chat")] }; @@ -2220,6 +2356,136 @@ function buildIosSimulatorPlan(args: string[]): CliPlan { return { kind: "execute", label: `ios-sim ${sub}`, steps: [actionStep("result", "ios_simulator", sub, collectGenericObjectArgs(args))] }; } +function readTrailingCommand(args: string[]): string | null { + const index = args.indexOf("--"); + if (index < 0) return null; + const command = args.slice(index + 1).join(" ").trim(); + args.splice(index); + return command.length ? command : null; +} + +function buildAppControlPlan(args: string[]): CliPlan { + const sub = firstPositional(args) ?? "status"; + if (sub === "help") return { kind: "help", text: buildAppControlHelp(args) }; + const numericPositionals = () => args.filter((value) => /^\d+(\.\d+)?$/.test(value)); + const readCoordinate = (flag: string, index: number): number => { + const value = readNumberOption(args, [flag]) ?? Number(numericPositionals()[index]); + if (!Number.isFinite(value)) throw new CliUsageError(`${flag} is required and must be a number.`); + return value; + }; + if (sub === "actions") return { kind: "execute", label: "App Control actions", steps: [listActionsStep("actions", "app_control")] }; + if (sub === "status") return { kind: "execute", label: "App Control status", steps: [actionStep("result", "app_control", "getStatus", collectGenericObjectArgs(args))] }; + if (sub === "logs" || sub === "log" || sub === "read" || sub === "tail") { + return { kind: "execute", label: "terminal read", steps: [actionStep("result", "app_control", "readTerminal", collectGenericObjectArgs(args, { + maxBytes: readIntOption(args, ["--max-bytes"], undefined), + since: readIntOption(args, ["--since"], undefined), + }))] }; + } + if (sub === "terminal") { + const mode = firstPositional(args) ?? "read"; + if (mode === "read" || mode === "logs" || mode === "tail") { + return { kind: "execute", label: "terminal read", steps: [actionStep("result", "app_control", "readTerminal", collectGenericObjectArgs(args, { + maxBytes: readIntOption(args, ["--max-bytes"], undefined), + since: readIntOption(args, ["--since"], undefined), + }))] }; + } + if (mode === "write" || mode === "send" || mode === "input") { + const data = readValue(args, ["--data", "--value", "--text"]) ?? args.join(" "); + if (!data.length) throw new CliUsageError("data is required."); + return { kind: "execute", label: "terminal write", steps: [actionStep("result", "app_control", "writeTerminal", collectGenericObjectArgs(args, { data }))] }; + } + if (mode === "signal" || mode === "interrupt" || mode === "stop") { + return { kind: "execute", label: "terminal signal", steps: [actionStep("result", "app_control", "signalTerminal", collectGenericObjectArgs(args, { + signal: readValue(args, ["--signal"]) ?? (mode === "stop" ? "SIGTERM" : "SIGINT"), + }))] }; + } + throw new CliUsageError("app-control terminal supports read, write, or signal."); + } + if (sub === "launch" || sub === "open" || sub === "start") { + const trailingCommand = readTrailingCommand(args); + const command = readValue(args, ["--command", "--cmd"]) ?? trailingCommand; + const appKind = readValue(args, ["--kind", "--app-kind"]) ?? "electron"; + const projectRoot = readValue(args, ["--project-root", "--root"]); + const laneId = readValue(args, ["--lane", "--lane-id"]); + const cwd = readValue(args, ["--cwd", "--working-directory"]); + const debugPort = readNumberOption(args, ["--debug-port", "--port"]); + const cdpPort = readNumberOption(args, ["--cdp-port"]); + const label = readValue(args, ["--label", "--name"]); + const chatSessionId = readValue(args, ["--chat-session", "--chat-session-id", "--session", "--session-id"]) ?? process.env.ADE_CHAT_SESSION_ID; + const force = readFlag(args, ["--force", "-f"]) ? true : undefined; + const positionalCommand = args.filter((arg) => arg !== "--" && !arg.startsWith("-")).join(" ").trim(); + const launchCommand = command ?? (positionalCommand.length ? positionalCommand : null); + if (!launchCommand) throw new CliUsageError("app-control launch requires a command, for example: ade app-control launch --command \"pnpm dev\"."); + return { + kind: "execute", + label: "App Control launch", + steps: [actionStep("result", "app_control", "launch", collectGenericObjectArgs(args, { + appKind, + projectRoot, + laneId, + command: launchCommand, + cwd, + debugPort, + cdpPort, + label, + chatSessionId, + force, + }))], + }; + } + if (sub === "connect" || sub === "attach") { + return { kind: "execute", label: "App Control connect", steps: [actionStep("result", "app_control", "connect", collectGenericObjectArgs(args, { + appKind: readValue(args, ["--kind", "--app-kind"]) ?? "electron", + projectRoot: readValue(args, ["--project-root", "--root"]), + cdpPort: readNumberOption(args, ["--cdp-port", "--port"]) ?? Number(numericPositionals()[0]), + label: readValue(args, ["--label", "--name"]), + chatSessionId: readValue(args, ["--chat-session", "--session"]) ?? process.env.ADE_CHAT_SESSION_ID, + force: readFlag(args, ["--force", "-f"]) ? true : undefined, + }))] }; + } + if (sub === "stop" || sub === "shutdown" || sub === "teardown" || sub === "close") { + return { kind: "execute", label: "App Control stop", steps: [actionStep("result", "app_control", "stop", collectGenericObjectArgs(args, { force: readFlag(args, ["--force", "-f"]) ? true : undefined }))] }; + } + if (sub === "screenshot" || sub === "capture") { + return { kind: "execute", label: "App Control screenshot", steps: [actionStep("result", "app_control", "screenshot", collectGenericObjectArgs(args))] }; + } + if (sub === "snapshot" || sub === "screen" || sub === "elements") { + return { kind: "execute", label: "App Control snapshot", steps: [actionStep("result", "app_control", "getSnapshot", collectGenericObjectArgs(args, { projectRoot: readValue(args, ["--project-root", "--root"]) }))] }; + } + if (sub === "inspect" || sub === "hit-test" || sub === "hover") { + return { kind: "execute", label: "App Control inspect point", steps: [actionStep("result", "app_control", "inspectPoint", collectGenericObjectArgs(args, { + projectRoot: readValue(args, ["--project-root", "--root"]), + x: readCoordinate("--x", 0), + y: readCoordinate("--y", 1), + includeScreenshot: readFlag(args, ["--screenshot", "--include-screenshot"]), + }))] }; + } + if (sub === "select") { + return { kind: "execute", label: "App Control select", steps: [actionStep("result", "app_control", "selectPoint", collectGenericObjectArgs(args, { + projectRoot: readValue(args, ["--project-root", "--root"]), + x: readCoordinate("--x", 0), + y: readCoordinate("--y", 1), + }))] }; + } + if (sub === "click" || sub === "tap") { + return { kind: "execute", label: "App Control click", steps: [actionStep("result", "app_control", "click", collectGenericObjectArgs(args, { + x: readCoordinate("--x", 0), + y: readCoordinate("--y", 1), + }))] }; + } + if (sub === "type" || sub === "text") { + return { kind: "execute", label: "App Control type", steps: [actionStep("result", "app_control", "typeText", collectGenericObjectArgs(args, { + text: requireValue( + readValue(args, ["--value", "--message", "--input-text"]) + ?? readCommandTextValue(args, ["--text"]) + ?? args.filter((arg) => arg !== "--text").join(" "), + "text", + ), + }))] }; + } + return { kind: "execute", label: `app-control ${sub}`, steps: [actionStep("result", "app_control", sub, collectGenericObjectArgs(args))] }; +} + function buildMemoryPlan(args: string[]): CliPlan { const sub = firstPositional(args) ?? "search"; if (sub === "actions") return { kind: "execute", label: "memory actions", steps: [listActionsStep("actions", "memory")] }; @@ -2617,12 +2883,14 @@ const VALUE_CARRIER_FLAGS: ReadonlySet = new Set([ "--automation", "--autonomy", "--backend", "--base", "--base-branch", "--base-ref", "--body", "--branch", "--branch-name", "--branch-ref", "--bundle", "--bundle-id", "--category", "--color", "--cols", "--command", "--comment", "--comment-id", "--commit", "--compare-ref", - "--caption", "--compare-to", "--content", "--context-file", "--cwd", "--data", + "--caption", "--cdp-port", "--chat-session", "--chat-session-id", "--compare-to", "--content", "--context-file", "--cwd", "--data", + "--debug-port", "--depth", "--desc", "--device", "--duration", "--duration-ms", "--description", "--domain", "--droid-autonomy", "--droid-permission-mode", "--duration-sec", "--enabled", "--event", "--end-x", "--end-y", "--file", "--fps", "--from", "--from-file", "--group", "--group-id", "--head", "--icon", "--id", "--index", "--input", "--input-json", "--input-text", "--instructions", + "--kind", "--json-input", "--lane", "--lane-id", "--limit", "--max-bytes", "--line", "--max-log-bytes", "--max-prompt-chars", "--max-rounds", "--memory", @@ -2630,15 +2898,15 @@ const VALUE_CARRIER_FLAGS: ReadonlySet = new Set([ "--model-id", "--name", "--new", "--new-path", "--number", "--old", "--old-path", "--owner", "--owner-id", "--owner-kind", "--params-json", "--parent", "--parent-lane", "--parent-lane-id", - "--path", "--permission-mode", "--permissions", "--pr", "--pr-id", + "--path", "--permission-mode", "--permissions", "--port", "--pr", "--pr-id", "--pr-number", "--pr-url", "--process", "--process-id", "--project-root", "--prompt", "--provider", "--pty", "--pty-id", "--query", "--question", "--reason", "--reasoning", "--recent-limit", "--ref", "--role", "--root", "--root-lane", "--round", "--rounds", "--rows", "--rule", "--run", "--run-id", "--scalar", "--scalar-json", "--scope", "--seconds", "--session", "--session-id", "--set", - "--set-json", "--sha", "--source", "--source-lane", "--stack", "--stack-id", + "--set-json", "--sha", "--signal", "--since", "--source", "--source-lane", "--stack", "--stack-id", "--scheme", "--start-point", "--start-x", "--start-y", "--stash-ref", "--step", "--step-id", "--suite", "--suite-id", "--surface", - "--tab", "--tab-identifier", "--target", "--target-id", "--thread", "--thread-id", "--timeout", "--timeout-ms", "--title", "--tool-type", + "--tab", "--tab-identifier", "--target", "--target-id", "--terminal", "--terminal-id", "--thread", "--thread-id", "--timeout", "--timeout-ms", "--title", "--tool-type", "--udid", "--url", "--value", "--workspace", "--workspace-id", "--workspace-root", "--x", "--xcodeproj", "--y", ]); @@ -2676,6 +2944,7 @@ function buildCliPlan(command: string[]): CliPlan { process: "run", processes: "run", pty: "shell", + term: "terminal", chats: "chat", work: "chat", agents: "agent", @@ -2686,6 +2955,9 @@ function buildCliPlan(command: string[]): CliPlan { artifacts: "proof", ios: "ios-sim", simulator: "ios-sim", + app: "app-control", + apps: "app-control", + electron: "app-control", setting: "settings", config: "settings", action: "actions", @@ -2697,6 +2969,9 @@ function buildCliPlan(command: string[]): CliPlan { if (primaryHelpKey === "ios-sim") { return { kind: "help", text: buildIosSimulatorHelp(args) }; } + if (primaryHelpKey === "app-control") { + return { kind: "help", text: buildAppControlHelp(args) }; + } return { kind: "help", text: HELP_BY_COMMAND[primaryHelpKey] ?? TOP_LEVEL_HELP }; } if (primary === "help") { @@ -2705,6 +2980,9 @@ function buildCliPlan(command: string[]): CliPlan { if (key === "ios-sim") { return { kind: "help", text: buildIosSimulatorHelp(args) }; } + if (key === "app-control") { + return { kind: "help", text: buildAppControlHelp(args) }; + } return { kind: "help", text: key && HELP_BY_COMMAND[key] ? HELP_BY_COMMAND[key] : TOP_LEVEL_HELP }; } if (primary === "version" || primary === "--version" || primary === "-v") { @@ -2746,6 +3024,7 @@ function buildCliPlan(command: string[]): CliPlan { if (primary === "prs" || primary === "pr") return buildPrPlan(args); if (primary === "run" || primary === "process" || primary === "processes") return buildRunPlan(args); if (primary === "shell" || primary === "pty") return buildShellPlan(args); + if (primary === "terminal" || primary === "term") return buildTerminalPlan(args); if (primary === "chat" || primary === "chats" || primary === "work") return buildChatPlan(args); if (primary === "agent" || primary === "agents") return buildAgentPlan(args); if (primary === "cto") return buildCtoPlan(args); @@ -2759,6 +3038,7 @@ function buildCliPlan(command: string[]): CliPlan { return buildProofPlan(args); } if (primary === "ios-sim" || primary === "ios" || primary === "simulator") return buildIosSimulatorPlan(args); + if (primary === "app-control" || primary === "app" || primary === "apps" || primary === "electron") return buildAppControlPlan(args); if (primary === "memory") return buildMemoryPlan(args); if (primary === "settings" || primary === "config" || primary === "setting") return buildSettingsPlan(args); if (primary === "actions" || primary === "action") return buildActionsPlan(args); @@ -3840,6 +4120,114 @@ function formatIosSimPreview(value: unknown): string { ]); } +function formatAppControlStatus(value: unknown): string { + const status = isRecord(value) ? value : {}; + const providers = Array.isArray(status.providers) ? status.providers.filter(isRecord) : []; + const session = isRecord(status.activeSession) + ? status.activeSession + : typeof status.status === "string" && status.label ? status : {}; + return [ + renderKeyValues("ADE App Control", [ + ["supported", status.supported], + ["platform", status.platform], + ["active app", session.label], + ["session", session.id], + ["status", session.status], + ["cdp port", session.cdpPort], + ["terminal", session.terminalSessionId], + ["pty", session.terminalPtyId], + ["chat session", session.chatSessionId], + ["pid", session.pid], + ["command", session.command], + ["error", session.lastError], + ]), + "", + renderTable( + ["provider", "ready", "detail"], + providers.map((provider) => [provider.provider, provider.available ? "yes" : "no", provider.detail]), + "Providers\n(none)", + ), + ].join("\n"); +} + +function formatAppControlSnapshot(value: unknown): string { + const snapshot = isRecord(value) ? value : {}; + const screenshot = isRecord(snapshot.screenshot) ? snapshot.screenshot : snapshot; + const screen = isRecord(snapshot.screen) ? snapshot.screen : {}; + const providers = Array.isArray(snapshot.providers) ? snapshot.providers.filter(isRecord) : []; + const elements = Array.isArray(snapshot.elements) ? snapshot.elements.filter(isRecord) : []; + const providerSummary = providers.map((provider) => `${provider.provider}:${provider.available ? provider.elementCount ?? "ok" : "unavailable"}`).join(", "); + return [ + renderKeyValues("ADE App Control snapshot", [ + ["title", snapshot.title], + ["url", snapshot.url], + ["captured", snapshot.capturedAt], + ["screenshot", screenshot.width && screenshot.height ? `${screenshot.width}x${screenshot.height}` : null], + ["screen", screen.width && screen.height ? `${screen.width}x${screen.height} @${screen.scale ?? 1}x` : null], + ["elements", elements.length], + ["providers", providerSummary], + ]), + elements.length ? "" : "", + elements.length + ? renderTable( + ["ref", "role", "label", "selector"], + elements.slice(0, 24).map((element) => [ + element.ref ?? element.id, + element.role ?? element.tagName, + element.label ?? element.value ?? element.testId, + element.selector, + ]), + "", + ) + : "", + ].filter(Boolean).join("\n"); +} + +function formatTerminalList(value: unknown): string { + const terminals = Array.isArray(value) + ? value.filter(isRecord) + : isRecord(value) && value.terminalId + ? [value] + : firstArray(value, ["terminals", "items"]); + return renderTable( + ["terminal", "pty", "chat", "status", "runtime", "title"], + terminals.map((terminal) => [ + terminal.terminalId, + terminal.ptyId, + terminal.chatSessionId, + terminal.status, + terminal.runtimeState, + terminal.title, + ]), + "ADE chat terminals\n(no terminals found)", + ); +} + +function formatTerminalRead(value: unknown): string { + const result = isRecord(value) ? value : {}; + const data = typeof result.data === "string" ? result.data : ""; + const header = renderKeyValues("ADE terminal scrollback", [ + ["terminal", result.terminalId], + ["nextSince", result.nextSince], + ["bytes", Buffer.byteLength(data, "utf8")], + ]); + return data.length ? `${header}\n\n${data}` : `${header}\n\n(no output)`; +} + +function formatAppControlSelection(value: unknown): string { + const item = firstRecord(value, ["item", "selection"]) ?? (isRecord(value) ? value : {}); + const metadata = isRecord(item.metadata) ? item.metadata : {}; + const selected = isRecord(metadata.selectedElement) ? metadata.selectedElement : {}; + return renderKeyValues("ADE App Control selection", [ + ["component", item.componentId], + ["source", isRecord(value) ? value.source ?? item.provider : item.provider], + ["file", item.sourceFile ? `${item.sourceFile}${item.sourceLine ? `:${item.sourceLine}` : ""}` : null], + ["selector", selected.selector], + ["label", selected.label ?? metadata.label], + ["selected", item.selectedAt], + ]); +} + function formatTextOutput(value: unknown, formatter: FormatterId | undefined): string { if (typeof value === "string") return value; if (isRecord(value) && typeof value.visual === "string" && (!formatter || formatter === "lanes")) return value.visual; @@ -3953,6 +4341,16 @@ function formatTextOutput(value: unknown, formatter: FormatterId | undefined): s return formatIosSimSelection(value); case "ios-sim-preview": return formatIosSimPreview(value); + case "app-control-status": + return formatAppControlStatus(value); + case "app-control-snapshot": + return formatAppControlSnapshot(value); + case "app-control-selection": + return formatAppControlSelection(value); + case "terminal-list": + return formatTerminalList(value); + case "terminal-read": + return formatTerminalRead(value); case "actions-list": return formatActionsList(value); case "automation-run-detail": @@ -3991,6 +4389,11 @@ function inferFormatter(plan: CliPlan & { kind: "execute" }): FormatterId | unde if (label === "ios simulator screen snapshot" || label === "ios simulator inspector snapshot" || label === "ios simulator screenshot") return "ios-sim-snapshot"; if (label === "ios simulator select" || label === "ios simulator inspect point") return "ios-sim-selection"; if (label === "ios simulator preview status" || label === "ios simulator previews" || label === "ios simulator preview render" || label === "ios simulator preview open") return "ios-sim-preview"; + if (label === "app control status" || label === "app control launch" || label === "app control connect" || label === "app control stop") return "app-control-status"; + if (label === "app control snapshot" || label === "app control screenshot") return "app-control-snapshot"; + if (label === "app control select" || label === "app control inspect point") return "app-control-selection"; + if (label === "terminal list" || label === "terminal active") return "terminal-list"; + if (label === "terminal read") return "terminal-read"; if (label === "actions list") return "actions-list"; if (label.endsWith("actions")) return "actions-list"; return "action-result"; diff --git a/apps/desktop/scripts/dev.cjs b/apps/desktop/scripts/dev.cjs index 0f77dcd42..ac598a1ac 100644 --- a/apps/desktop/scripts/dev.cjs +++ b/apps/desktop/scripts/dev.cjs @@ -31,7 +31,23 @@ function canListenOnHost(port, host) { }); } +function somethingIsListening(port, host) { + return new Promise((resolve) => { + const socket = net.createConnection({ port, host }); + socket.setTimeout(150); + const settle = (busy) => { + socket.destroy(); + resolve(busy); + }; + socket.once("connect", () => settle(true)); + socket.once("timeout", () => settle(false)); + socket.once("error", () => settle(false)); + }); +} + async function isPortFree(port) { + if (await somethingIsListening(port, "127.0.0.1")) return false; + if (await somethingIsListening(port, "::1")) return false; const ipv4Free = await canListenOnHost(port, "127.0.0.1"); if (!ipv4Free) return false; const ipv6Free = await canListenOnHost(port, "::1"); @@ -180,9 +196,17 @@ function terminateChild(child, signal) { async function main() { const devPort = await choosePort(5173, 32); const devServerUrl = `http://localhost:${devPort}`; - const remoteDebugPort = Number.parseInt(process.env.ADE_ELECTRON_REMOTE_DEBUGGING_PORT || "9222", 10); + const remoteDebugPortRaw = + process.env.ADE_APP_CONTROL_CDP_PORT + || process.env.ADE_APP_CONTROL_REMOTE_DEBUGGING_PORT + || process.env.ADE_ELECTRON_REMOTE_DEBUGGING_PORT + || "9222"; + const remoteDebugPort = Number.parseInt(remoteDebugPortRaw, 10); if (!Number.isFinite(remoteDebugPort) || remoteDebugPort <= 0) { - throw new Error(`Invalid ADE_ELECTRON_REMOTE_DEBUGGING_PORT: ${process.env.ADE_ELECTRON_REMOTE_DEBUGGING_PORT ?? ""}`); + throw new Error(`Invalid Electron remote debugging port: ${remoteDebugPortRaw}`); + } + if (process.env.ADE_APP_CONTROL_CDP_PORT) { + process.stdout.write(`[ade] honoring ADE App Control CDP port ${remoteDebugPort}\n`); } process.stdout.write(`[ade] dev launcher using ${devServerUrl}\n`); process.stdout.write(`[ade] electron CDP endpoint: http://127.0.0.1:${remoteDebugPort}/json/version\n`); diff --git a/apps/desktop/src/main/main.ts b/apps/desktop/src/main/main.ts index 5bf4021b7..a8d56a722 100644 --- a/apps/desktop/src/main/main.ts +++ b/apps/desktop/src/main/main.ts @@ -1,4 +1,4 @@ -import { app, BrowserWindow, dialog, Menu, MenuItem, nativeImage, protocol, safeStorage, shell } from "electron"; +import { app, BrowserWindow, dialog, nativeImage, protocol, safeStorage, shell } from "electron"; import os from "node:os"; import path from "node:path"; import { pathToFileURL } from "node:url"; @@ -137,6 +137,7 @@ import { createMissionBudgetService } from "./services/orchestrator/missionBudge import { transitionMissionStatus } from "./services/orchestrator/missionLifecycle"; import { createComputerUseArtifactBrokerService } from "./services/computerUse/computerUseArtifactBrokerService"; import { createIosSimulatorService } from "./services/ios/iosSimulatorService"; +import { createAppControlService } from "./services/appControl/appControlService"; import { createSyncService } from "./services/sync/syncService"; import { ApnsService, ApnsKeyStore } from "./services/notifications/apnsService"; import { @@ -2710,6 +2711,30 @@ app.whenReady().then(async () => { onEvent: (payload) => emitProjectEvent(projectRoot, IPC.iosSimulatorEvent, payload), }); + const appControlService = createAppControlService({ + projectRoot, + logger, + ptyService, + resolveLaneId: async ({ cwd, projectRoot: requestedProjectRoot, laneId, chatSessionId }) => { + const explicitLaneId = laneId?.trim(); + if (explicitLaneId) return explicitLaneId; + const chatId = chatSessionId?.trim(); + if (chatId) { + const chatSession = await agentChatService.getSessionSummary(chatId).catch(() => null); + if (chatSession?.laneId) return chatSession.laneId; + } + const targetRoot = path.resolve(cwd || requestedProjectRoot || projectRoot); + const lanes = await laneService.list({ includeArchived: false }); + const matchingLane = lanes.find((lane) => { + const worktreePath = path.resolve(lane.worktreePath); + const attachedRootPath = lane.attachedRootPath ? path.resolve(lane.attachedRootPath) : null; + return targetRoot === worktreePath || targetRoot.startsWith(`${worktreePath}${path.sep}`) || targetRoot === attachedRootPath; + }); + return matchingLane?.id ?? lanes[0]?.id ?? null; + }, + onEvent: (payload) => + emitProjectEvent(projectRoot, IPC.appControlEvent, payload), + }); missionPreflightService = createMissionPreflightService({ logger, projectRoot, @@ -3384,6 +3409,7 @@ app.whenReady().then(async () => { automationPlannerService, computerUseArtifactBrokerService, iosSimulatorService, + appControlService, orchestratorService, aiOrchestratorService, missionBudgetService, @@ -3548,6 +3574,7 @@ app.whenReady().then(async () => { ptyService, computerUseArtifactBrokerService, iosSimulatorService, + appControlService, automationService, automationPlannerService, githubService, @@ -3597,6 +3624,7 @@ app.whenReady().then(async () => { prPollingService, computerUseArtifactBrokerService, iosSimulatorService, + appControlService, queueLandingService, issueInventoryService, prSummaryService, @@ -3702,6 +3730,7 @@ app.whenReady().then(async () => { agentChatService: null, computerUseArtifactBrokerService: null, iosSimulatorService: null, + appControlService: null, githubService: null, feedbackReporterService: null, prService: null, @@ -3901,6 +3930,11 @@ app.whenReady().then(async () => { } catch { // ignore } + try { + ctx.appControlService?.dispose?.(); + } catch { + // ignore + } try { ctx.testService.disposeAll(); } catch { diff --git a/apps/desktop/src/main/services/adeActions/registry.ts b/apps/desktop/src/main/services/adeActions/registry.ts index 42cb52dc1..71ea0f7c5 100644 --- a/apps/desktop/src/main/services/adeActions/registry.ts +++ b/apps/desktop/src/main/services/adeActions/registry.ts @@ -46,11 +46,13 @@ export const ADE_ACTION_DOMAIN_NAMES = [ "file", "process", "pty", + "terminal", "layout", "tiling_tree", "graph_state", "computer_use_artifacts", "ios_simulator", + "app_control", "automations", "issue", ] as const; @@ -336,11 +338,13 @@ export const ADE_ACTION_ALLOWLIST: Partial; + write(args?: unknown): unknown; + signal(args?: unknown): unknown; + activeForChat(args?: unknown): unknown; +}; + +function buildTerminalDomainService(runtime: AdeRuntime): TerminalDomainService | null { + if (!runtime.ptyService) return null; + return { + list(args) { + return runtime.ptyService.listTerminals(args as Parameters[0]); + }, + read(args) { + return runtime.ptyService.readTerminal(args as Parameters[0]); + }, + write(args) { + return runtime.ptyService.writeTerminal(args as Parameters[0]); + }, + signal(args) { + return runtime.ptyService.signalTerminal(args as Parameters[0]); + }, + activeForChat(args) { + return runtime.ptyService.activeForChat(args as Parameters[0]); + }, + }; +} + export function getAdeActionDomainServices( runtime: AdeRuntime, ): Partial> { @@ -595,11 +628,13 @@ export function getAdeActionDomainServices( file: toService(runtime.fileService), process: toService(runtime.processService), pty: toService(runtime.ptyService), + terminal: toService(buildTerminalDomainService(runtime)), layout: toService(buildLayoutDomainService(runtime)), tiling_tree: toService(buildTilingTreeDomainService(runtime)), graph_state: toService(buildGraphStateDomainService(runtime)), computer_use_artifacts: toService(runtime.computerUseArtifactBrokerService), ios_simulator: toService(runtime.iosSimulatorService), + app_control: toService(runtime.appControlService), automations: toService(buildAutomationsDomainService(runtime)), issue: toService(buildIssueDomainService(runtime)), }; diff --git a/apps/desktop/src/main/services/appControl/appControlService.test.ts b/apps/desktop/src/main/services/appControl/appControlService.test.ts new file mode 100644 index 000000000..3477e510d --- /dev/null +++ b/apps/desktop/src/main/services/appControl/appControlService.test.ts @@ -0,0 +1,215 @@ +import type { EventEmitter } from "node:events"; +import { beforeEach, describe, expect, it, vi } from "vitest"; +import type { Logger } from "../logging/logger"; + +type FakeCdpTarget = { + id: string; + type: string; + title?: string; + url?: string; + webSocketDebuggerUrl?: string; +}; + +const mockState = vi.hoisted(() => ({ + httpResponses: [] as Array>, + sockets: [] as Array<{ url: string; sent: string[] }>, + runtimeValues: [] as unknown[], +})); + +vi.mock("node:http", async () => { + const { EventEmitter } = await import("node:events"); + return { + default: { + get: (_url: string, _options: { timeout?: number }, callback: (response: EventEmitter & { statusCode?: number }) => void) => { + const request = new EventEmitter() as EventEmitter & { + destroy: (error?: Error) => void; + setTimeout: () => void; + }; + request.destroy = (error?: Error) => { + if (error) request.emit("error", error); + }; + request.setTimeout = () => {}; + const responseTargets = mockState.httpResponses.shift(); + queueMicrotask(async () => { + try { + const targets = await responseTargets; + const response = new EventEmitter() as EventEmitter & { statusCode?: number }; + response.statusCode = 200; + callback(response); + response.emit("data", Buffer.from(JSON.stringify(targets ?? []))); + response.emit("end"); + } catch (error) { + request.emit("error", error); + } + }); + return request; + }, + }, + }; +}); + +vi.mock("ws", async () => { + const { EventEmitter } = await import("node:events"); + class FakeWebSocket extends EventEmitter { + static readonly OPEN = 1; + static readonly CLOSING = 2; + static readonly CLOSED = 3; + + readonly OPEN = FakeWebSocket.OPEN; + readonly CLOSING = FakeWebSocket.CLOSING; + readonly CLOSED = FakeWebSocket.CLOSED; + + readyState = FakeWebSocket.OPEN; + readonly sent: string[] = []; + + constructor(readonly url: string) { + super(); + mockState.sockets.push({ url, sent: this.sent }); + queueMicrotask(() => this.emit("open")); + } + + send(payload: string, callback?: (error?: Error) => void): void { + this.sent.push(payload); + const message = JSON.parse(payload) as { id: number; method: string }; + if (this.readyState === FakeWebSocket.OPEN) { + const result = message.method === "Runtime.evaluate" + ? { result: { value: mockState.runtimeValues.shift() ?? {} } } + : {}; + this.emit("message", Buffer.from(JSON.stringify({ id: message.id, result }))); + } + callback?.(); + } + + close(): void { + this.readyState = FakeWebSocket.CLOSED; + this.emit("close"); + } + + terminate(): void { + this.close(); + } + } + + return { WebSocket: FakeWebSocket }; +}); + +import { createAppControlService } from "./appControlService"; + +function createLogger(): Logger { + return { + debug: vi.fn(), + info: vi.fn(), + warn: vi.fn(), + error: vi.fn(), + }; +} + +function deferred(): { promise: Promise; resolve: (value: T) => void; reject: (error: Error) => void } { + let resolve!: (value: T) => void; + let reject!: (error: Error) => void; + const promise = new Promise((resolvePromise, rejectPromise) => { + resolve = resolvePromise; + reject = rejectPromise; + }); + return { promise, resolve, reject }; +} + +function target(id: string): FakeCdpTarget { + return { + id, + type: "page", + title: `Window ${id}`, + url: `app://test/?view=${id}`, + webSocketDebuggerUrl: `ws://127.0.0.1/devtools/page/${id}`, + }; +} + +describe("appControlService", () => { + beforeEach(() => { + vi.useFakeTimers(); + mockState.httpResponses.length = 0; + mockState.sockets.length = 0; + mockState.runtimeValues.length = 0; + }); + + it("lets manual target switches win over an in-flight health poll", async () => { + const targetA = target("a"); + const targetB = target("b"); + mockState.httpResponses.push([targetA, targetB]); + + const service = createAppControlService({ + projectRoot: "/tmp/project", + logger: createLogger(), + }); + + await service.connect({ cdpPort: 12345, force: true }); + expect(service.getStatus().activeSession?.cdpTargetId).toBe("a"); + + const healthPoll = deferred(); + mockState.httpResponses.push(healthPoll.promise); + vi.advanceTimersByTime(2_000); + await Promise.resolve(); + + mockState.httpResponses.push([targetA, targetB]); + const attached = await service.attachToTarget("b"); + expect(attached.cdpTargetId).toBe("b"); + expect(service.getStatus().activeSession?.cdpTargetId).toBe("b"); + + healthPoll.resolve([targetA, targetB]); + await Promise.resolve(); + + expect(service.getStatus().activeSession?.cdpTargetId).toBe("b"); + expect(service.getStatus().activeSession?.cdpEndpoint).toBe(targetB.webSocketDebuggerUrl); + }); + + it("dispatches clicks without a blocking mouseMoved prelude", async () => { + const targetA = target("a"); + mockState.httpResponses.push([targetA]); + + const service = createAppControlService({ + projectRoot: "/tmp/project", + logger: createLogger(), + }); + + await service.connect({ cdpPort: 12345, force: true }); + const socket = mockState.sockets.at(-1); + expect(socket).toBeTruthy(); + socket!.sent.length = 0; + + await service.click({ x: 20, y: 40, scale: 2 }); + + const mouseEvents = socket!.sent + .map((payload) => JSON.parse(payload) as { method: string; params?: { type?: string; x?: number; y?: number } }) + .filter((message) => message.method === "Input.dispatchMouseEvent"); + expect(mouseEvents.map((event) => event.params?.type)).toEqual(["mousePressed", "mouseReleased"]); + expect(mouseEvents.map((event) => ({ x: event.params?.x, y: event.params?.y }))).toEqual([ + { x: 10, y: 20 }, + { x: 10, y: 20 }, + ]); + }); + + it("uses an in-page click fallback when the Electron target is hidden", async () => { + const targetA = target("a"); + mockState.httpResponses.push([targetA]); + mockState.runtimeValues.push( + { hasFocus: true, visibilityState: "hidden" }, + { ok: true, target: "button", label: "Open full app" }, + ); + + const service = createAppControlService({ + projectRoot: "/tmp/project", + logger: createLogger(), + }); + + await service.connect({ cdpPort: 12345, force: true }); + const socket = mockState.sockets.at(-1); + expect(socket).toBeTruthy(); + socket!.sent.length = 0; + + await service.click({ x: 20, y: 40, scale: 2 }); + + const messages = socket!.sent.map((payload) => JSON.parse(payload) as { method: string }); + expect(messages.filter((message) => message.method === "Runtime.evaluate")).toHaveLength(2); + expect(messages.some((message) => message.method === "Input.dispatchMouseEvent")).toBe(false); + }); +}); diff --git a/apps/desktop/src/main/services/appControl/appControlService.ts b/apps/desktop/src/main/services/appControl/appControlService.ts new file mode 100644 index 000000000..5e954129b --- /dev/null +++ b/apps/desktop/src/main/services/appControl/appControlService.ts @@ -0,0 +1,2026 @@ +import { randomUUID } from "node:crypto"; +import fs from "node:fs"; +import http from "node:http"; +import net from "node:net"; +import path from "node:path"; +import { WebSocket, type RawData } from "ws"; +import type { + AppControlClickArgs, + AppControlConnectArgs, + AppControlContextItem, + AppControlElement, + AppControlEventPayload, + AppControlFrame, + AppControlInspectPointArgs, + AppControlInspectResult, + AppControlLaunchArgs, + AppControlScreencastFrame, + AppControlScreenshot, + AppControlSelectResult, + AppControlSession, + AppControlSnapshot, + AppControlSnapshotArgs, + AppControlSourceMatch, + AppControlStatus, + AppControlStopArgs, + AppControlTarget, + AppControlTypeTextArgs, +} from "../../../shared/types"; +import type { Logger } from "../logging/logger"; +import type { createPtyService } from "../pty/ptyService"; + +const CDP_POLL_MS = 500; +const CDP_HEALTH_POLL_MS = 2_000; +const CDP_COMMAND_TIMEOUT_MS = 15_000; +const MAX_DOM_ELEMENTS = 450; +const SOURCE_FILE_EXTENSIONS = new Set([".ts", ".tsx", ".js", ".jsx", ".mjs", ".cjs", ".html", ".css"]); +const SOURCE_SKIP_DIRS = new Set([".git", ".ade", "node_modules", "dist", "build", "out", "coverage", ".next", ".vite"]); +const sourceFileCache = new Map(); + +type CreateAppControlServiceArgs = { + projectRoot: string; + logger: Logger; + ptyService?: ReturnType | null; + resolveLaneId?: (args: { + projectRoot: string; + cwd: string; + laneId?: string | null; + chatSessionId?: string | null; + }) => Promise | string | null; + onEvent?: ((payload: AppControlEventPayload) => void) | null; +}; + +type CdpTarget = { + id: string; + type: string; + title?: string; + url?: string; + webSocketDebuggerUrl?: string; +}; + +type CdpScreenshotResponse = { + data: string; +}; + +type CdpRuntimeEvaluateResponse = { + result?: { + type?: string; + value?: T; + description?: string; + }; + exceptionDetails?: unknown; +}; + +type CdpInputPageState = { + hasFocus: boolean; + visibilityState: string; +}; + +type CdpDomClickResult = { + ok: boolean; + target: string | null; + label: string | null; +}; + +type CdpPendingCommand = { + resolve: (value: unknown) => void; + reject: (error: Error) => void; + timer: NodeJS.Timeout; +}; + +type DomSnapshotPayload = { + url: string | null; + title: string | null; + viewport: { width: number; height: number; devicePixelRatio: number }; + elements: Array<{ + tagName: string | null; + role: string | null; + label: string | null; + value: string | null; + selector: string | null; + testId: string | null; + rect: AppControlFrame; + metadata: Record; + }>; +}; + +type ResolvedLaunch = { + label: string; + cwd: string; + commandForDisplay: string; +}; + +function nowIso(): string { + return new Date().toISOString(); +} + +function round(value: number): number { + return Math.round(value * 100) / 100; +} + +function roundFrame(frame: AppControlFrame): AppControlFrame { + return { + x: round(frame.x), + y: round(frame.y), + width: round(frame.width), + height: round(frame.height), + }; +} + +function shellQuote(value: string): string { + if (/^[A-Za-z0-9_/:=.,@%+-]+$/.test(value)) return value; + return `'${value.replace(/'/g, "'\\''")}'`; +} + +function commandForwardsAppControlDebug(command: string): boolean { + return /\{ADE_APP_CONTROL_DEBUG_FLAGS\}|\bADE_APP_CONTROL_(?:DEBUG_FLAGS|CDP_PORT|REMOTE_DEBUGGING_PORT)\b|--remote-debugging-port\b/.test(command); +} + +function commandLooksLikePackageScriptLaunch(command: string): boolean { + return /(?:^|[;&|]\s*)(?:[A-Za-z_][A-Za-z0-9_]*=(?:"[^"]*"|'[^']*'|[^\s;&|]+)\s+)*(?:(?:npm|pnpm|yarn|bun)\s+(?:run\s+)?[A-Za-z0-9:_./-]+)\s*$/.test(command.trim()); +} + +function commandLooksLikeDirectElectronLaunch(command: string): boolean { + return /(?:^|[;&|]\s*)(?:[A-Za-z_][A-Za-z0-9_]*=(?:"[^"]*"|'[^']*'|[^\s;&|]+)\s+)*(?:npx\s+)?electron(?:\s+[^;&|]*)?\s*$/.test(command.trim()); +} + +function unquoteShellValue(value: string): string { + const trimmed = value.trim(); + if ( + (trimmed.startsWith("'") && trimmed.endsWith("'")) + || (trimmed.startsWith("\"") && trimmed.endsWith("\"")) + ) { + return trimmed.slice(1, -1); + } + return trimmed; +} + +function insertDebugFlagsIntoDirectElectronCommand(command: string, debugFlags: string[]): string { + const flags = debugFlags.map(shellQuote).join(" "); + return command.replace( + /((?:^|[;&|]\s*)(?:[A-Za-z_][A-Za-z0-9_]*=(?:"[^"]*"|'[^']*'|[^\s;&|]+)\s+)*(?:npx\s+)?electron)(?=\s|$)/, + `$1 ${flags}`, + ); +} + +function prependEnvToShellSegments(script: string, envPrefix: string): string { + const trimmedEnv = envPrefix.trim(); + if (!trimmedEnv) return script; + return script + .split(/(\s*&&\s*)/) + .map((segment) => { + if (/^\s*&&\s*$/.test(segment)) return segment; + const trimmed = segment.trim(); + return trimmed ? `${trimmedEnv} ${trimmed}` : segment; + }) + .join(""); +} + +function rewritePackageScriptElectronLaunch(command: string, debugFlags: string[], fallbackCwd: string): string | null { + const match = command.trim().match(/^(?.*?)(?(?:[A-Za-z_][A-Za-z0-9_]*=(?:"[^"]*"|'[^']*'|[^\s;&|]+)\s+)*)(?npm|pnpm|yarn|bun)\s+(?:run\s+)?(?