From 590c91e96c32f7fc71a7ee83dd9019e223d306af Mon Sep 17 00:00:00 2001 From: devswha Date: Thu, 5 Nov 2026 16:36:44 +0900 Subject: [PATCH 1/4] feat(terminal): draw herdr-core panes with libghostty, Kitty pictures included, behind a Settings toggle --- DESIGN.md | 10 +- THIRD_PARTY_NOTICES.md | 99 ++++ changelog.d/terminal-ghostty-renderer.md | 5 + docs/development.md | 1 + scripts/ci-browser.sh | 1 + scripts/compressed-assets.ts | 2 +- scripts/ghostty-vt/build.sh | 25 + scripts/ghostty-vt/kitty-on-wasm.patch | 40 ++ scripts/terminal-ghostty-demo-regression.ts | 167 +++++++ server/static.test.ts | 2 +- server/static.ts | 5 +- src/components/PaneTerminal.css | 13 + src/components/PaneTerminal.tsx | 5 +- src/components/SettingsDialog.tsx | 3 + src/lib/ghostty/capture.ts | 81 +++ src/lib/ghostty/ghostty-vt.wasm | Bin 0 -> 1110716 bytes src/lib/ghostty/overlay.ts | 270 ++++++++++ src/lib/ghostty/vt.ts | 521 ++++++++++++++++++++ src/lib/i18n/settings.ja.ts | 2 + src/lib/i18n/settings.ko.ts | 2 + src/lib/i18n/settings.zh.ts | 2 + src/lib/settings.legacy-fixture.ts | 2 +- src/lib/settings.test.ts | 20 + src/lib/settings.ts | 19 +- src/lib/useTerminalMount.ts | 7 +- src/lib/ws.ts | 5 + 26 files changed, 1297 insertions(+), 12 deletions(-) create mode 100644 changelog.d/terminal-ghostty-renderer.md create mode 100755 scripts/ghostty-vt/build.sh create mode 100644 scripts/ghostty-vt/kitty-on-wasm.patch create mode 100644 scripts/terminal-ghostty-demo-regression.ts create mode 100644 src/lib/ghostty/capture.ts create mode 100644 src/lib/ghostty/ghostty-vt.wasm create mode 100644 src/lib/ghostty/overlay.ts create mode 100644 src/lib/ghostty/vt.ts diff --git a/DESIGN.md b/DESIGN.md index 8ce3b5774..85eb958c5 100644 --- a/DESIGN.md +++ b/DESIGN.md @@ -3154,7 +3154,8 @@ Alert rows of the table above from the `--droplet-*` roles in `src/styles.css`. 640px the New tab button (off by default), the Artifacts chip (off by default, the device setting `composerArtifacts`), and on a touch screen the suggestion chip (off by default); then **Quick replies**; then **File viewer**: wrap long lines, Markdown width, Diff layout), **Terminal** (font size, font family, wheel - speed, input mode, Key bar, Clipboard from a pane, Use alongside herdr's own window, Follow + speed, input mode, Key bar, Clipboard from a pane, Draw with libghostty (off by default, the + device setting `terminalGhostty`), Use alongside herdr's own window, Follow herdr's focus), **Alerts**, **Voice input**, **Agents**, **Shortcuts** (the platform-resolved global bindings), **Phone & devices** (Connect a phone, Keep screen on, Install; then Portal, where the server offers it; then paired devices; then Access), @@ -3453,6 +3454,13 @@ Alert rows of the table above from the `--droplet-*` roles in `src/styles.css`. ordered change set, so Retry resends the current change, never an old request body. ### Terminal host, key bar and drawer +- **Draw with libghostty** (Settings > Terminal, off by default, herdr-core only): libghostty-vt draws the + pane's text, cursor and Kitty graphics pictures in a layer over xterm's screen, at xterm's cell size; + xterm's own text and cursor are hidden under it and xterm keeps input, selection, links, search and + scrolling. It follows Ghostty's own split of a terminal core (libghostty-vt) from the renderer an app + draws with, rather than a second terminal widget: everything built on xterm keeps working while the + drawing moves. The layer takes no pointer events; pictures keep the layer Kitty gave them (below + the text, or above it). - **Find in terminal** in the header's More menu and Mod+Shift+F opens a find bar above the grid. It uses the existing input and icon-button primitives: a query, match ordinal, previous/next and close controls, with a dim note that scrolling is shared by all clients. diff --git a/THIRD_PARTY_NOTICES.md b/THIRD_PARTY_NOTICES.md index 24de62768..ac9bf80c0 100644 --- a/THIRD_PARTY_NOTICES.md +++ b/THIRD_PARTY_NOTICES.md @@ -1518,6 +1518,105 @@ FROM, OUT OF THE USE OR INABILITY TO USE THE FONT SOFTWARE OR FROM OTHER DEALINGS IN THE FONT SOFTWARE. ``` +## libghostty-vt + +`src/lib/ghostty/ghostty-vt.wasm` is libghostty-vt from [Ghostty](https://github.com/ghostty-org/ghostty) at +commit e95bed6, built for wasm32 by `scripts/ghostty-vt/build.sh` with the small change in +`scripts/ghostty-vt/kitty-on-wasm.patch` (Kitty graphics on, file media off). It draws a terminal when +Settings > Terminal > Draw with libghostty is on, under the MIT License below. The module links two +packages Ghostty's build brings in: [Wuffs](https://github.com/google/wuffs) 0.4.0-alpha.10, which decodes +Kitty PNG pictures, under the MIT License (it is offered under MIT or Apache-2.0), and +[uucode](https://github.com/jacobsandlund/uucode) 0.2.0, Unicode tables, under the MIT License. + +``` +MIT License + +Copyright (c) 2024 Mitchell Hashimoto, Ghostty contributors + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. +``` + +Wuffs: + +``` +Copyright 2023 The Wuffs Authors + +Permission is hereby granted, free of charge, to any +person obtaining a copy of this software and associated +documentation files (the "Software"), to deal in the +Software without restriction, including without +limitation the rights to use, copy, modify, merge, +publish, distribute, sublicense, and/or sell copies of +the Software, and to permit persons to whom the Software +is furnished to do so, subject to the following +conditions: + +The above copyright notice and this permission notice +shall be included in all copies or substantial portions +of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF +ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED +TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A +PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY +CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR +IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +DEALINGS IN THE SOFTWARE. +``` + +uucode: + +``` +# uucode license + +MIT License + +Copyright (c) 2026 Jacob Sandlund + +Permission is hereby granted, free of charge, to any person obtaining a copy of +this software and associated documentation files (the "Software"), to deal in +the Software without restriction, including without limitation the rights to +use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies +of the Software, and to permit persons to whom the Software is furnished to do +so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + +## Other licenses + +See [./licenses](./licenses) for licenses of code being used in the repo: + +* [LICENSE_Bjoern_Hoehrmann](./licenses/LICENSE_Bjoern_Hoehrmann) +* [LICENSE_unicode](./licenses/LICENSE_unicode) +``` + ## JetBrains Mono `src/fonts/JetBrainsMono-Regular.woff2` and `src/fonts/JetBrainsMono-Bold.woff2` are diff --git a/changelog.d/terminal-ghostty-renderer.md b/changelog.d/terminal-ghostty-renderer.md new file mode 100644 index 000000000..3afb4accc --- /dev/null +++ b/changelog.d/terminal-ghostty-renderer.md @@ -0,0 +1,5 @@ +### Added +- Settings > Terminal > Draw with libghostty (off by default) draws a pane with libghostty, the + terminal engine of Ghostty, on a PC that runs herdr-core, as the desktop app does, with the pictures + a program sends with the Kitty graphics protocol (kitty icat, timg, chafa). Typing, selection, + links and search still go through xterm.js underneath. diff --git a/docs/development.md b/docs/development.md index c32628a08..86429b210 100644 --- a/docs/development.md +++ b/docs/development.md @@ -266,6 +266,7 @@ bun scripts/omo-plan-browser-regression.ts # OmO todo progress and plan panel on bun scripts/pane-input-focus-demo-regression.ts # the input focus shortcut against fictional panes bun scripts/mirror-scroll-demo-regression.ts # fixed mirror wheel/touch, native history, no guessed program input, observe guards bun scripts/core-history-scroll-demo-regression.ts # a herdr-core server's history kept in xterm: wheel and swipe scroll it, a replay erases the last one, nothing typed; herdr's own keeps none (#2663) +bun scripts/terminal-ghostty-demo-regression.ts # Settings -> Draw with libghostty: a herdr-core pane drawn from its first byte, a Kitty picture placed and deleted, typing still through xterm; herdr's own and off stay xterm without fetching the module bun scripts/mirror-fold-demo-regression.ts # a mirror, or an attached pane a phone watches at a wider grid, folded whole (portrait, landscape), Original width from the pane menu, driving phones and desktops unfolded bun scripts/shortcut-exact-modifiers-demo-regression.ts # recording exact-modifier chords: confirm, refusals, dispatch bun scripts/output-browser-qa.ts # terminal output flow control end to end diff --git a/scripts/ci-browser.sh b/scripts/ci-browser.sh index dc3952f1f..2f0e2eb2f 100644 --- a/scripts/ci-browser.sh +++ b/scripts/ci-browser.sh @@ -79,6 +79,7 @@ run_script scripts/terminal-touch-burst-regression.ts run_script scripts/omo-question-wheel-regression.ts run_script scripts/mirror-scroll-demo-regression.ts run_script scripts/core-history-scroll-demo-regression.ts +run_script scripts/terminal-ghostty-demo-regression.ts run_script scripts/mirror-fold-demo-regression.ts run_script scripts/chat-no-rewind-demo-regression.ts run_script scripts/terminal-cjk-punctuation-regression.ts diff --git a/scripts/compressed-assets.ts b/scripts/compressed-assets.ts index d6a1b8f9c..97c3a7274 100644 --- a/scripts/compressed-assets.ts +++ b/scripts/compressed-assets.ts @@ -9,7 +9,7 @@ const brotli = promisify(brotliCompress), gz = promisify(gzip); export function compressedAssets(): Plugin { return { name: "compressed-static-assets", async writeBundle(options, bundle) { if (!options.dir) return; - await Promise.all(Object.values(bundle).filter(asset => /\.(?:js|css)$/.test(asset.fileName)).map(async asset => { + await Promise.all(Object.values(bundle).filter(asset => /\.(?:js|css|wasm)$/.test(asset.fileName)).map(async asset => { const source = asset.type === "chunk" ? asset.code : asset.source; const bytes = typeof source === "string" ? Buffer.from(source) : Buffer.from(source); const [br, gzip] = await Promise.all([brotli(bytes, { params: { [constants.BROTLI_PARAM_QUALITY]: 5 } }), gz(bytes, { level: 9 })]); diff --git a/scripts/ghostty-vt/build.sh b/scripts/ghostty-vt/build.sh new file mode 100755 index 000000000..81ab176bd --- /dev/null +++ b/scripts/ghostty-vt/build.sh @@ -0,0 +1,25 @@ +#!/usr/bin/env bash +# Rebuilds src/lib/ghostty/ghostty-vt.wasm: libghostty-vt from ghostty (MIT) at a pinned commit, with +# kitty-on-wasm.patch, which turns the Kitty graphics protocol on for wasm32-freestanding (upstream +# leaves it off there: no file medium and no PNG decoder). The patch refuses the file, temporary +# file and shared memory media, which have no meaning in a browser, and links ghostty's bundled +# wuffs PNG decoder, so a PNG a program sends is decoded inside the module. +# +# ZIG=/path/to/zig-0.17.1/zig scripts/ghostty-vt/build.sh +# +# Expected output sha256 (zig 0.17.1, x86_64-linux): +# b8e8d0f8dd4e84718002198f80d5399005d6d42f2d419b31644343d5703a1444 +set -euo pipefail +GHOSTTY_COMMIT=e95bed6 +here="$(cd "$(dirname "$0")" && pwd)" +repo="$(cd "$here/../.." && pwd)" +zig="${ZIG:-zig}" +work="$(mktemp -d)" +trap 'rm -rf "$work"' EXIT +git -C "$work" init -q ghostty +git -C "$work/ghostty" fetch -q --depth 1 https://github.com/ghostty-org/ghostty "$GHOSTTY_COMMIT" +git -C "$work/ghostty" checkout -q FETCH_HEAD +git -C "$work/ghostty" apply "$here/kitty-on-wasm.patch" +(cd "$work/ghostty" && "$zig" build -Demit-lib-vt -Dtarget=wasm32-freestanding -Doptimize=ReleaseSmall --prefix "$work/out") +cp "$work/out/bin/ghostty-vt.wasm" "$repo/src/lib/ghostty/ghostty-vt.wasm" +sha256sum "$repo/src/lib/ghostty/ghostty-vt.wasm" diff --git a/scripts/ghostty-vt/kitty-on-wasm.patch b/scripts/ghostty-vt/kitty-on-wasm.patch new file mode 100644 index 000000000..8c900c0c6 --- /dev/null +++ b/scripts/ghostty-vt/kitty-on-wasm.patch @@ -0,0 +1,40 @@ +diff --git a/src/terminal/build_options.zig b/src/terminal/build_options.zig +index 628ecbd..1542c32 100644 +--- a/src/terminal/build_options.zig ++++ b/src/terminal/build_options.zig +@@ -272,7 +272,7 @@ pub const Options = struct { + /// targets, so it is always disabled there regardless of the + /// feature setting. + pub fn kittyGraphics(self: Options, target: std.Target) bool { +- if (target.os.tag == .freestanding) return false; ++ _ = target; + return self.features.kitty_graphics; + } + +diff --git a/src/terminal/kitty/graphics_image.zig b/src/terminal/kitty/graphics_image.zig +index 281bb02..cba04a5 100644 +--- a/src/terminal/kitty/graphics_image.zig ++++ b/src/terminal/kitty/graphics_image.zig +@@ -182,6 +182,9 @@ pub const LoadingImage = struct { + } + } + ++ // Freestanding (wasm) has no filesystem: only direct payloads. ++ if (comptime builtin.target.os.tag == .freestanding) return error.UnsupportedMedium; ++ + // Depending on the medium, load the data from the path. + switch (t.medium) { + .direct => unreachable, // handled above +diff --git a/src/terminal/sys.zig b/src/terminal/sys.zig +index 3f6b501..712a754 100644 +--- a/src/terminal/sys.zig ++++ b/src/terminal/sys.zig +@@ -16,7 +16,7 @@ const build_options = @import("terminal_options"); + /// and the exact semantics are up to callers. For example, the Kitty Graphics + /// Protocol will work but cannot accept PNG images. + pub var decode_png: ?DecodePngFn = png: { +- if (build_options.artifact == .lib) break :png null; ++ if (build_options.artifact == .lib and @import("builtin").target.os.tag != .freestanding) break :png null; + break :png &decodePngWuffs; + }; + diff --git a/scripts/terminal-ghostty-demo-regression.ts b/scripts/terminal-ghostty-demo-regression.ts new file mode 100644 index 000000000..a57e90ecf --- /dev/null +++ b/scripts/terminal-ghostty-demo-regression.ts @@ -0,0 +1,167 @@ +/** + * Settings -> Terminal -> Draw with libghostty: on a herdr-core server (`terminal-history`) libghostty + * draws the pane over its xterm from every byte xterm was given, the ones before its module loaded + * included; a Kitty graphics picture is drawn where it was placed and goes when the program deletes + * it; typing still goes through xterm. herdr's own server, or the setting off, keep xterm alone and + * never fetch the module. Demo fixtures only: no herdr, hc or user state. + */ +import assert from "node:assert/strict"; +import { mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { chromium, type Browser, type Page } from "playwright-core"; +import panes from "../site/demo/fixtures/panes.json"; +import { buildDemoApp } from "./demo-build.ts"; +import { chromePath } from "./chrome-path.ts"; + +interface Frame { type: string; pane_id?: string; data?: string; text?: string; features?: string[] } +interface GhosttyFixture { inputs: Frame[]; print(data: string): void } +declare global { interface Window { ghosttyFixture: GhosttyFixture } } + +/** The demo serves the pane; its features and the pane's bytes are the test's. */ +function installGhosttyFixture(): void { + const core = new URL(location.href).searchParams.get("server") === "core"; + // printed at attach, before libghostty's module can have loaded: it comes from the held bytes + const first = "\x1b[H\x1b[2JBEFORE-LOAD\r\n"; + const sockets: Array): void }> = []; + let pane: string | null = null; + const state: GhosttyFixture = { + inputs: [], + print: (data) => { for (const socket of sockets) if (pane) socket.push({ type: "pty-data", fixture: "ghostty", pane_id: pane, data }); }, + }; + window.ghosttyFixture = state; + const Demo = window.WebSocket; + const Recorded = function (url: string | URL, protocols?: string | string[]) { + const socket = new Demo(url, protocols) as (typeof sockets)[number]; + sockets.push(socket); + const push = socket.push.bind(socket); + socket.push = (frame) => { + if ((frame.type === "snapshot" || frame.type === "features") && Array.isArray(frame.features) && core) { + push({ ...frame, features: [...(frame.features as string[]), "terminal-history"] }); + } else if (frame.type === "pty-data") { + // the demo's own recording would move the screen under the test + if (frame.fixture !== "ghostty") return; + const { fixture: _, ...output } = frame; + push(output); + } else push(frame); + }; + const send = socket.send.bind(socket); + socket.send = (raw) => { + const frame = JSON.parse(String(raw)) as Frame; + if (["input", "keys", "submit"].includes(frame.type)) state.inputs.push(frame); + send(raw); + if (frame.type === "attach" && frame.pane_id) { + pane = frame.pane_id; + socket.push({ type: "pty-data", fixture: "ghostty", pane_id: pane, data: first }); + } + }; + return socket; + }; + for (const name of ["CONNECTING", "OPEN", "CLOSING", "CLOSED"] as const) Object.defineProperty(Recorded, name, { value: Demo[name] }); + window.WebSocket = Recorded as unknown as typeof WebSocket; +} + +/** a 4x4 square of one colour as a Kitty graphics picture, raw RGBA sent in the escape itself, shown 4 columns by 2 rows */ +const kittySquare = (id: number, rgb: [number, number, number]): string => { + const pixels = Buffer.alloc(4 * 4 * 4); + for (let i = 0; i < 16; i++) pixels.set([...rgb, 255], i * 4); + return `\x1b_Ga=T,f=32,s=4,v=4,i=${id},c=4,r=2,q=2;${pixels.toString("base64")}\x1b\\`; +}; + +const settled = (page: Page) => page.evaluate(() => new Promise((resolve) => requestAnimationFrame(() => requestAnimationFrame(() => resolve())))); +const print = async (page: Page, data: string, marker: string) => { + await page.evaluate((data) => window.ghosttyFixture.print(data), `${data}${marker}\r\n`); + await page.waitForFunction((marker) => [...document.querySelectorAll(".pane-frame.is-current .ghostty-row")].some((row) => row.textContent?.includes(marker)), marker); + await settled(page); +}; +/** the colour at the middle of the pane's one Kitty picture, or null when it draws none */ +const picture = (page: Page) => page.evaluate(() => { + const canvases = [...document.querySelectorAll(".pane-frame.is-current .ghostty-image")]; + if (canvases.length !== 1) return canvases.length === 0 ? null : { count: canvases.length }; + const canvas = canvases[0]!; + const [r, g, b] = canvas.getContext("2d")!.getImageData(canvas.width >> 1, canvas.height >> 1, 1, 1).data; + return { rgb: [r, g, b], width: canvas.style.width, top: canvas.style.top }; +}); + +async function open(browser: Browser, server: number, prefix: string, kind: "core" | "herdr", ghostty: boolean): Promise<{ page: Page; wasm: string[]; errors: string[] }> { + const context = await browser.newContext({ viewport: { width: 1280, height: 900 }, locale: "en-US" }); + await context.addInitScript((ghostty) => { + localStorage.setItem("ovis:settings", JSON.stringify({ language: "en", defaultView: "terminal", terminalInputMode: "direct", alertsOn: false })); + localStorage.setItem("ovis:terminal-ghostty", JSON.stringify(ghostty)); + }, ghostty); + const page = await context.newPage(); + page.setDefaultTimeout(10000); + const wasm: string[] = []; + const errors: string[] = []; + page.on("request", (request) => { if (request.url().endsWith(".wasm")) wasm.push(request.url()); }); + page.on("pageerror", (error) => errors.push(error.message)); + page.on("console", (message) => { if (message.type() === "warning" && message.text().includes("libghostty")) errors.push(message.text()); }); + await page.goto(`http://127.0.0.1:${server}${prefix}?pane=${encodeURIComponent(panes.shell)}&view=terminal&server=${kind}`); + await page.locator(".pane-frame.is-current .xterm-rows", { hasText: "BEFORE-LOAD" }).waitFor(); + return { page, wasm, errors }; +} + +const app = mkdtempSync(join(tmpdir(), "terminal-ghostty-demo-")); +try { + await buildDemoApp(app); + const index = join(app, "index.html"); + writeFileSync(index, readFileSync(index, "utf8").replace(/\n\n