fix(computer-use): show file panels and fresh looks in the same call (#7065)

Two round trips cost an extra call on most saves and refusals:

- A Save or Open panel that the agent's own step opened came back with the step as
  file_panel_approval_required. The agent then had to observe again so the broker could apply
  the organization's file rule. That rule now runs on the step's own look: by default the panel
  is shown in the same receipt, an organization that asks still asks, and one that blocks still
  keeps the agent out.
- An act the helper refused before any step started (a dialog opened, the window changed or
  another window came in front), or one refused because the helper session was renewed, came
  back with no look. These now carry a fresh observation, as refusals inside a batch already
  did, so the next call can act. Nothing refused is ever sent again.

Signed-off-by: Jalil <reachjalil@gmail.com>

Author: reachjalil

apps/desktop/electron/computer-control.test.mjs

@@ -100,11 +100,17 @@ test("follow-ups and idle cleanup renew without approvals or replaying old input
   await f.control.endTurn({ sessionId: "chat" });
   assert.equal(payload(await f.control.execute({ sessionId: "chat", name: "computer_observe", args: {} })).ok, true);
   await new Promise(resolve => setTimeout(resolve, 35));
+  const looks = f.calls.filter(name => name === "computer_observe").length;
   const result = await f.control.execute({ sessionId: "chat", name: "computer_act", args: { observation_id: "old", action: { type: "click", x: 1, y: 1 } } });
   assert.equal(payload(result).code, "observation_required");
   assert.equal(f.calls.filter(name => name === "computer_open_session").length, 3);
   assert.ok(!f.calls.includes("computer_act"));
   assert.deepEqual(f.selected, []);
+  // The refusal carries a look at the renewed window, so the next call can act on it.
+  const [, post, view] = result.content.map(part => JSON.parse(part.text));
+  assert.deepEqual(post, { post_observation: { ok: true }, actions_replayed: false, next: "act", note: "Nothing was sent. Act on this fresh observation." });
+  assert.equal(view.observation_id, "observation");
+  assert.equal(f.calls.filter(name => name === "computer_observe").length, looks + 1);
 });
 
 test("human takeover survives end-of-turn and idle cleanup until Continue", async t => {
@@ -507,6 +513,14 @@ test("an act refused before anything was sent returns a fresh observation instea
   await act({ action: { type: "key", key: "return" }, screenshot: false });
   assert.equal(seen[1]?.args.include_image, false, "the model's screenshot choice still holds");
 
+  // A refusal the helper gives before any step starts carries no step status: it gets a look too.
+  refusal = { ok: false, code: "observation_required", message: "A dialog opened or closed since the last observation. Observe again; nothing was sent.", next: "observe" };
+  await act({ action: { type: "press", ref: "e2" } });
+  assert.deepEqual(seen.map((call) => call.name), ["computer_act", "computer_observe"], "a dialog the last step opened is looked at in this call");
+  refusal = { ok: false, code: "protected_input", message: "Take over for this step.", next: "human_takeover" };
+  await act({ action: { type: "type", text: "x" } });
+  assert.deepEqual(seen.map((call) => call.name), ["computer_act"], "a refusal that hands over gets no look");
+
   refusal = { ...unsent, code: "scope_denied", next: "open_session" };
   await act({ action: { type: "click", x: 1, y: 1 } });
   assert.deepEqual(seen.map((call) => call.name), ["computer_act"], "only refusals that ask for a new look get one");
@@ -1567,18 +1581,22 @@ test("the agent saves what it sees as an image under the same file rule as save_
 /** A Mac helper with an Open panel in front: it asks before showing the panel, and picks the file once its path is approved. */
 function openPanelFixture(options = {}) {
   const seen = [];
+  // As the helper does, a panel once allowed stays allowed until it closes; picking a file closes it.
+  let open = false, allowed = false;
   const f = fixture({ seen, platform: "darwin", ...options, toolResult: (name) => {
     const args = seen.at(-1).args;
-    const looks = seen.filter((call) => call.name === "computer_observe").length;
-    if (name === "computer_observe" && looks > 1 && args.file_panel_access !== 31) {
-      return text({ ok: false, code: "file_panel_approval_required", app: "Photoshop", panel_id: 31, dispatched: false, next: "observe" });
+    if (name === "computer_observe" && open && !allowed) {
+      if (args.file_panel_access !== 31) return text({ ok: false, code: "file_panel_approval_required", app: "Photoshop", panel_id: 31, dispatched: false, next: "observe" });
+      allowed = true;
     }
     if (name !== "computer_act") return undefined;
-    return text(JSON.stringify(args.file_approval) === JSON.stringify([args.action.path])
-      ? { ok: true, status: "dispatched", path: "file_chosen" }
+    const chosen = JSON.stringify(args.file_approval) === JSON.stringify([args.action.path]);
+    if (chosen) open = false;
+    return text(chosen ? { ok: true, status: "dispatched", path: "file_chosen" }
       : { ok: false, code: "files_approval_required", files: [args.action.path], open: true, app: "Photoshop", dispatched: false, next: "observe" });
   } });
-  const look = () => f.control.execute({ sessionId: "chat", name: "computer_observe", args: {} });
+  // Each explicit look is at a panel the app has just opened.
+  const look = () => { open = true; allowed = false; return f.control.execute({ sessionId: "chat", name: "computer_observe", args: {} }); };
   const pick = (path) => f.control.execute({ sessionId: "chat", name: "computer_act", args: { observation_id: "observation", action: { type: "open_in_panel", path } } });
   const cards = () => f.requests.filter((request) => request?.reason === "files");
   const acts = () => seen.filter((call) => call.name === "computer_act").map((call) => call.args.file_approval ?? null);
@@ -1589,6 +1607,46 @@ function openPanelFixture(options = {}) {
   const answer = async (count, reply) => f.control.answerRequest({ sessionId: "chat", answer: reply, id: (await card(count)).id });
   return { ...f, look, pick, cards, acts, card, answer };
 }
+/** A Mac helper whose next act opens a Save panel: looks after it ask first, as the helper does for any file panel. */
+function savePanelFixture(movePolicy) {
+  const seen = [];
+  let opened = false;
+  const f = fixture({ seen, platform: "darwin", movePolicy, toolResult: (name) => {
+    const args = seen.at(-1).args;
+    if (name === "computer_act") { opened = true; return text({ ok: true, status: "dispatched", completed: 1, next: "observe", steps: [{ index: 0, action: "key", status: "dispatched", path: "menu_for_shortcut" }] }); }
+    if (name === "computer_observe" && opened && args.file_panel_access !== 31) {
+      return text({ ok: false, code: "file_panel_approval_required", app: "TextEdit", panel_id: 31, dispatched: false, next: "observe" });
+    }
+    return undefined;
+  } });
+  const looks = () => seen.filter((call) => call.name === "computer_observe").map((call) => call.args.file_panel_access ?? null);
+  const save = () => f.control.execute({ sessionId: "chat", name: "computer_act", args: { observation_id: "observation", action: { type: "key", key: "s", modifiers: ["command"] } } });
+  const open = () => f.control.setFeatureEnabled(true).then(() => f.control.execute({ sessionId: "chat", name: "computer_open", args: { app_id: "editor", mode: "control", purpose: "Save the letter" } }));
+  return { ...f, looks, save, open };
+}
+
+test("a Save panel the agent's own step opened is shown in that step's receipt, with no second look", async (t) => {
+  const asked = [];
+  const f = savePanelFixture(async (input) => { asked.push(input); return { effect: "allow" }; }); t.after(() => f.control.reset(true));
+  await f.open();
+  const before = f.looks().length;
+  const parts = (await f.save()).content.map((part) => JSON.parse(part.text));
+  assert.deepEqual(parts[1], { post_observation: { ok: true }, actions_replayed: false }, "the panel is looked at in the same call");
+  assert.deepEqual(f.looks().slice(before), [null, 31], "the second look carries the panel the organization's rule allowed");
+  assert.deepEqual(asked, [{ kind: "computer_files", sessionId: "chat" }]);
+  assert.deepEqual(f.requests.filter((request) => request?.reason === "files"), [], "by default nothing asks");
+});
+
+test("an organization that blocks file panels still keeps the agent out of one its step opened", async (t) => {
+  const f = savePanelFixture(async () => ({ effect: "deny", message: "Blocked here." })); t.after(() => f.control.reset(true));
+  await f.open();
+  const before = f.looks().length;
+  const parts = (await f.save()).content.map((part) => JSON.parse(part.text));
+  assert.equal(parts[1].post_observation.code, "file_panel_approval_required");
+  assert.equal(parts.length, 2, "no view of the panel");
+  assert.deepEqual(f.looks().slice(before), [null], "never looked at with access");
+});
+
 const openForPlacing = (f) => f.control.setFeatureEnabled(true).then(() => f.control.execute({ sessionId: "chat", name: "computer_open", args: { app_id: "editor", mode: "control", purpose: "Place the map" } }));
 
 test("an Open panel is looked through and a file picked in it without a card by default", async (t) => {

packages/computer-use/computer-control.mjs

@@ -42,6 +42,8 @@ const withPostObservation = (receipt, status, content = [], extra = {}) => ({ ..
   { type: "text", text: JSON.stringify({ post_observation: status, actions_replayed: false,
     ...(!status.ok ? { next: status.next, fresh_observation_required: true } : {}), ...extra }) }, ...content] });
 const COORDINATE_ACTIONS = new Set(["move", "click", "double_click", "triple_click", "scroll", "drag"]);
+/** Refusals before any input that a fresh look answers: the receipt carries that look, so the next call can act. */
+const LOOK_AGAIN = new Set(["observation_required", "stale_observation", "focus_lost", "window_not_frontmost"]);
 const actionEntries = (args) => [args.action, ...(Array.isArray(args.actions) ? args.actions : [])].filter((entry) => entry && typeof entry === "object");
 /** Command-C, Command-X or Command-V: a copy, cut or paste through the agent's clipboard. */
 const isClipboardStep = (entry) => entry.type === "key" && typeof entry.key === "string" && ["c", "x", "v"].includes(entry.key.toLowerCase())
@@ -1100,7 +1102,19 @@ export function createComputerControl({ adapters, runProgram, runScript, onTimin
           if (["background", "foreground"].includes(state.delivery)) current.delivery = state.delivery;
           invalidateObservation(current); current.needsObservation = true;
           updateSession(current, opened, args.purpose);
-          if (["computer_act", "computer_zoom"].includes(name)) return failure("observation_required", "The internal session was renewed. Observe the current window before input; nothing was dispatched.", { next: "observe", actions_replayed: false });
+          if (["computer_act", "computer_zoom"].includes(name)) {
+            const refused = failure("observation_required", "The internal session was renewed. Observe the current window before input; nothing was dispatched.", { next: "observe", actions_replayed: false });
+            // The model would observe next: it gets that look now, in this call, as after any refusal before input.
+            const look = await current.transport.callTool("computer_observe", { session_id: current.nativeId }, { signal: active });
+            progress();
+            const seen = stateOf(look);
+            if (look.isError || seen?.ok !== true || typeof seen.observation_id !== "string" || needsPerson(look)) return refused;
+            updateSession(current, look);
+            current.needsObservation = false;
+            const listed = listChanges(null, look.content);
+            current.listing = listed.listing;
+            return withPostObservation(refused, { ok: true }, listed.content, { next: "act", note: "Nothing was sent. Act on this fresh observation." });
+          }
         }
         if (!["computer_discover", "computer_open"].includes(name) && !current.nativeId) return failure("session_required", "No session is open. Call computer_open first. Nothing was dispatched.");
         if (name === "computer_open") {
@@ -1141,8 +1155,11 @@ export function createComputerControl({ adapters, runProgram, runScript, onTimin
         // The helper refused before sending anything and asks for a new look, such as when
         // the window image changed: the model would observe next, so it gets that look now.
         // The refused input is never retried; the model decides again from the fresh view.
-        const refusedUnsent = name === "computer_act" && !call.program && nativeResult !== undefined && state.ok === false
-          && state.status === "failed" && state.completed === 0 && state.may_have_acted === false && state.next === "observe";
+        // So does a refusal the helper gives before any step starts (it carries no step status):
+        // a dialog opened, the window changed, or another window came to the front.
+        const refusedUnsent = name === "computer_act" && !call.program && nativeResult !== undefined && state.ok === false && state.next === "observe"
+          && (state.status === "failed" && state.completed === 0 && state.may_have_acted === false
+            || state.status === undefined && state.may_have_acted === undefined && LOOK_AGAIN.has(state.code));
         if (current.nativeId && (inputLanded || refusedUnsent || name === "computer_open" && !result.isError && state.ok && state.state === "active")) {
           call.post = { current, generation, nativeId: current.nativeId, receipt: result, captured: false, refused: refusedUnsent,
             observationGeneration: name === "computer_open" ? current.observationGeneration : observationGeneration };
@@ -1175,12 +1192,26 @@ export function createComputerControl({ adapters, runProgram, runScript, onTimin
               // continuation across a person takeover. Watch frames are unrelated.
               // Unless the model chose, the computer decides whether an image adds anything.
               // Opening an app also lists its menu-bar commands, so the agent can run one at once.
-              const observation = await current.transport.callTool("computer_observe", { session_id: current.nativeId,
+              const look = { session_id: current.nativeId,
                 ...(call.postImage === undefined ? {} : { include_image: call.postImage }),
-                ...(name === "computer_open" && platform === "darwin" && current.adapter?.placement === "local" && !call.program ? { menus: true } : {}) }, { signal: active });
+                ...(name === "computer_open" && platform === "darwin" && current.adapter?.placement === "local" && !call.program ? { menus: true } : {}) };
+              let observation = await current.transport.callTool("computer_observe", look, { signal: active });
               progress(); assertUiActive(current);
               if (!postStillOwned(call)) return postWarning(result, "observation_withheld", "human_takeover");
-              const observed = stateOf(observation);
+              let observed = stateOf(observation);
+              // The action opened a file panel: the organization's rule decides here, as it does for the
+              // model's own look, so the agent sees the panel in this same call. By default nothing asks.
+              if (observed?.code === "file_panel_approval_required" && observed.dispatched === false && Number.isSafeInteger(observed.panel_id)) {
+                const checked = await approveMove(grant, { kind: "files", app: typeof observed.app === "string" ? observed.app : "the app", names: [] }, signal);
+                progress(); assertUiActive(current);
+                if (!postStillOwned(call)) return postWarning(result, "observation_withheld", "human_takeover");
+                if (checked.answer === "allow") {
+                  observation = await current.transport.callTool("computer_observe", { ...look, file_panel_access: observed.panel_id }, { signal: active });
+                  progress(); assertUiActive(current);
+                  if (!postStillOwned(call)) return postWarning(result, "observation_withheld", "human_takeover");
+                  observed = stateOf(observation);
+                }
+              }
               if (observation.isError || observed?.ok !== true || typeof observed.observation_id !== "string" || !observed.observation_id || needsPerson(observation)) {
                 current.needsObservation = true;
                 nativeStopped(current, observation); updateSession(current, observation);