From f963a71148b7c13f1bf156799cc9c1be4a634809 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Sat, 16 May 2026 16:25:07 -0400 Subject: [PATCH 1/3] Add bezeled screenshots and screen recording --- AGENTS.md | 2 + README.md | 4 + cli/XCWChromeRenderer.h | 3 + cli/XCWChromeRenderer.m | 105 +++++++++++++++ cli/XCWProcessRunner.h | 7 + cli/XCWProcessRunner.m | 20 ++- cli/XCWSimctl.h | 6 + cli/XCWSimctl.m | 81 ++++++++++- cli/native/XCWNativeBridge.h | 3 +- cli/native/XCWNativeBridge.m | 21 ++- client/src/api/controls.ts | 50 ++++++- client/src/app/AppShell.tsx | 72 ++++++++++ .../src/features/simulators/SimulatorMenu.tsx | 33 +++++ client/src/features/toolbar/Toolbar.tsx | 9 ++ docs/api/rest.md | 3 +- docs/cli/commands.md | 3 + docs/cli/flags.md | 3 +- docs/cli/index.md | 2 + docs/guide/architecture.md | 2 +- docs/guide/index.md | 2 +- docs/guide/testing.md | 4 +- docs/index.md | 2 +- packages/simdeck-test/dist/index.d.ts | 10 +- packages/simdeck-test/dist/index.js | 29 +++- packages/simdeck-test/src/index.ts | 49 ++++++- scripts/integration/cli.mjs | 44 ++++++ scripts/integration/js-api.mjs | 19 +++ server/native_stubs.c | 9 ++ server/src/api/routes.rs | 70 +++++++++- server/src/main.rs | 127 +++++++++++++++++- server/src/native/bridge.rs | 25 +++- server/src/native/ffi.rs | 6 + skills/simdeck/SKILL.md | 7 +- 33 files changed, 804 insertions(+), 28 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index e1a46833..a5026d2c 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -137,6 +137,8 @@ Useful direct commands: ./build/simdeck pasteboard set "hello" ./build/simdeck pasteboard get ./build/simdeck screenshot --output screen.png +./build/simdeck screenshot --with-bezel --output screen-bezel.png +./build/simdeck record --seconds 5 --output screen-recording.mp4 ./build/simdeck describe ./build/simdeck tap 120 240 ./build/simdeck tap --label "Continue" --wait-timeout-ms 5000 diff --git a/README.md b/README.md index 416270ef..fa9e63e6 100644 --- a/README.md +++ b/README.md @@ -130,6 +130,8 @@ simdeck toggle-appearance simdeck pasteboard set "hello" simdeck pasteboard get simdeck screenshot --output screen.png +simdeck screenshot --with-bezel --output screen-bezel.png +simdeck record --seconds 5 --output screen-recording.mp4 simdeck stream --frames 120 > stream.h264 simdeck describe simdeck describe --format agent --max-depth 4 @@ -206,6 +208,8 @@ try { await sim.tap(0.5, 0.5); await sim.waitFor({ label: "Continue" }); await sim.screenshot(); + await sim.screenshot({ withBezel: true }); + await sim.record({ seconds: 5 }); } finally { sim.close(); } diff --git a/cli/XCWChromeRenderer.h b/cli/XCWChromeRenderer.h index 64f7c513..2a30175e 100644 --- a/cli/XCWChromeRenderer.h +++ b/cli/XCWChromeRenderer.h @@ -15,6 +15,9 @@ NS_ASSUME_NONNULL_BEGIN error:(NSError * _Nullable * _Nullable)error; + (nullable NSData *)screenMaskPNGDataForDeviceName:(NSString *)deviceName error:(NSError * _Nullable * _Nullable)error; ++ (nullable NSData *)screenshotPNGDataForDeviceName:(NSString *)deviceName + screenPNGData:(NSData *)screenPNGData + error:(NSError * _Nullable * _Nullable)error; + (nullable NSDictionary *)profileForDeviceName:(NSString *)deviceName error:(NSError * _Nullable * _Nullable)error; diff --git a/cli/XCWChromeRenderer.m b/cli/XCWChromeRenderer.m index dea918ee..abd4feab 100644 --- a/cli/XCWChromeRenderer.m +++ b/cli/XCWChromeRenderer.m @@ -249,6 +249,111 @@ + (nullable NSData *)screenMaskPNGDataForDeviceName:(NSString *)deviceName return [self PNGDataForPDFAtPath:maskPath scale:1.0 error:error]; } ++ (nullable NSData *)screenshotPNGDataForDeviceName:(NSString *)deviceName + screenPNGData:(NSData *)screenPNGData + error:(NSError * _Nullable __autoreleasing *)error { + NSDictionary *profile = [self profileForDeviceName:deviceName error:error]; + if (profile == nil) { + return nil; + } + + NSData *chromePNGData = [self PNGDataForDeviceName:deviceName includeButtons:YES error:error]; + if (chromePNGData == nil) { + return nil; + } + + NSImage *screenImage = [[NSImage alloc] initWithData:screenPNGData]; + NSImage *chromeImage = [[NSImage alloc] initWithData:chromePNGData]; + if (screenImage == nil || chromeImage == nil) { + if (error != NULL) { + *error = [NSError errorWithDomain:XCWChromeRendererErrorDomain + code:15 + userInfo:@{ + NSLocalizedDescriptionKey: @"Unable to decode simulator screenshot or chrome PNG data.", + }]; + } + return nil; + } + + CGFloat scale = 3.0; + CGFloat totalWidth = [self numberValue:profile[@"totalWidth"]]; + CGFloat totalHeight = [self numberValue:profile[@"totalHeight"]]; + CGFloat screenX = [self numberValue:profile[@"screenX"]]; + CGFloat screenY = [self numberValue:profile[@"screenY"]]; + CGFloat screenWidth = [self numberValue:profile[@"screenWidth"]]; + CGFloat screenHeight = [self numberValue:profile[@"screenHeight"]]; + NSInteger pixelWidth = MAX((NSInteger)ceil(totalWidth * scale), 1); + NSInteger pixelHeight = MAX((NSInteger)ceil(totalHeight * scale), 1); + if (totalWidth <= 0.0 || totalHeight <= 0.0 || screenWidth <= 0.0 || screenHeight <= 0.0) { + if (error != NULL) { + *error = [NSError errorWithDomain:XCWChromeRendererErrorDomain + code:16 + userInfo:@{ + NSLocalizedDescriptionKey: @"Device chrome profile did not include usable screenshot geometry.", + }]; + } + return nil; + } + + NSBitmapImageRep *bitmap = [[NSBitmapImageRep alloc] initWithBitmapDataPlanes:NULL + pixelsWide:pixelWidth + pixelsHigh:pixelHeight + bitsPerSample:8 + samplesPerPixel:4 + hasAlpha:YES + isPlanar:NO + colorSpaceName:NSDeviceRGBColorSpace + bytesPerRow:0 + bitsPerPixel:32]; + if (bitmap == nil) { + if (error != NULL) { + *error = [NSError errorWithDomain:XCWChromeRendererErrorDomain + code:17 + userInfo:@{ + NSLocalizedDescriptionKey: @"Unable to create a bitmap for bezeled screenshot rendering.", + }]; + } + return nil; + } + + NSGraphicsContext *graphicsContext = [NSGraphicsContext graphicsContextWithBitmapImageRep:bitmap]; + [NSGraphicsContext saveGraphicsState]; + [NSGraphicsContext setCurrentContext:graphicsContext]; + graphicsContext.imageInterpolation = NSImageInterpolationHigh; + NSRect outputRect = NSMakeRect(0.0, 0.0, pixelWidth, pixelHeight); + [[NSColor clearColor] setFill]; + NSRectFillUsingOperation(outputRect, NSCompositingOperationClear); + + NSRect screenRect = NSMakeRect(screenX * scale, + pixelHeight - ((screenY + screenHeight) * scale), + screenWidth * scale, + screenHeight * scale); + NSDictionary *hints = @{ NSImageHintInterpolation: @(NSImageInterpolationHigh) }; + [screenImage drawInRect:screenRect + fromRect:NSZeroRect + operation:NSCompositingOperationSourceOver + fraction:1.0 + respectFlipped:NO + hints:hints]; + [chromeImage drawInRect:outputRect + fromRect:NSZeroRect + operation:NSCompositingOperationSourceOver + fraction:1.0 + respectFlipped:NO + hints:hints]; + [NSGraphicsContext restoreGraphicsState]; + + NSData *pngData = [bitmap representationUsingType:NSBitmapImageFileTypePNG properties:@{}]; + if (pngData.length == 0 && error != NULL) { + *error = [NSError errorWithDomain:XCWChromeRendererErrorDomain + code:18 + userInfo:@{ + NSLocalizedDescriptionKey: @"Unable to encode bezeled simulator screenshot PNG.", + }]; + } + return pngData.length > 0 ? pngData : nil; +} + + (nullable NSDictionary *)profileForChromeInfo:(NSDictionary *)chromeInfo error:(NSError * _Nullable __autoreleasing *)error { NSDictionary *plist = chromeInfo[@"plist"]; diff --git a/cli/XCWProcessRunner.h b/cli/XCWProcessRunner.h index 70626ab2..f76b8580 100644 --- a/cli/XCWProcessRunner.h +++ b/cli/XCWProcessRunner.h @@ -31,6 +31,13 @@ NS_ASSUME_NONNULL_BEGIN timeoutSec:(NSTimeInterval)timeoutSec error:(NSError * _Nullable * _Nullable)error; ++ (XCWProcessResult *)runLaunchPath:(NSString *)launchPath + arguments:(NSArray *)arguments + inputData:(nullable NSData *)inputData + timeoutSec:(NSTimeInterval)timeoutSec + timeoutSignal:(int)timeoutSignal + error:(NSError * _Nullable * _Nullable)error; + @end NS_ASSUME_NONNULL_END diff --git a/cli/XCWProcessRunner.m b/cli/XCWProcessRunner.m index fbf61028..5e4c2fde 100644 --- a/cli/XCWProcessRunner.m +++ b/cli/XCWProcessRunner.m @@ -119,6 +119,20 @@ + (XCWProcessResult *)runLaunchPath:(NSString *)launchPath inputData:(NSData *)inputData timeoutSec:(NSTimeInterval)timeoutSec error:(NSError * _Nullable __autoreleasing *)error { + return [self runLaunchPath:launchPath + arguments:arguments + inputData:inputData + timeoutSec:timeoutSec + timeoutSignal:SIGTERM + error:error]; +} + ++ (XCWProcessResult *)runLaunchPath:(NSString *)launchPath + arguments:(NSArray *)arguments + inputData:(NSData *)inputData + timeoutSec:(NSTimeInterval)timeoutSec + timeoutSignal:(int)timeoutSignal + error:(NSError * _Nullable __autoreleasing *)error { int stdoutFD = -1; int stderrFD = -1; int stdinPipe[2] = { -1, -1 }; @@ -244,8 +258,10 @@ + (XCWProcessResult *)runLaunchPath:(NSString *)launchPath } if (hasTimeout && [deadline timeIntervalSinceNow] <= 0) { timedOut = YES; - kill(pid, SIGTERM); - NSDate *killDeadline = [NSDate dateWithTimeIntervalSinceNow:2.0]; + int signalToSend = timeoutSignal > 0 ? timeoutSignal : SIGTERM; + kill(pid, signalToSend); + NSTimeInterval graceSeconds = signalToSend == SIGINT ? 10.0 : 2.0; + NSDate *killDeadline = [NSDate dateWithTimeIntervalSinceNow:graceSeconds]; do { waitResult = waitpid(pid, &waitStatus, WNOHANG); if (waitResult == pid || (waitResult < 0 && errno != EINTR)) { diff --git a/cli/XCWSimctl.h b/cli/XCWSimctl.h index 430821d3..d8a0932b 100644 --- a/cli/XCWSimctl.h +++ b/cli/XCWSimctl.h @@ -19,6 +19,12 @@ NS_ASSUME_NONNULL_BEGIN - (BOOL)openURL:(NSString *)urlString simulatorUDID:(NSString *)udid error:(NSError * _Nullable * _Nullable)error; - (BOOL)launchBundleID:(NSString *)bundleID simulatorUDID:(NSString *)udid error:(NSError * _Nullable * _Nullable)error; - (nullable NSData *)screenshotPNGForSimulatorUDID:(NSString *)udid error:(NSError * _Nullable * _Nullable)error; +- (nullable NSData *)screenshotPNGForSimulatorUDID:(NSString *)udid + includeBezel:(BOOL)includeBezel + error:(NSError * _Nullable * _Nullable)error; +- (nullable NSData *)screenRecordingMP4ForSimulatorUDID:(NSString *)udid + durationSeconds:(NSTimeInterval)durationSeconds + error:(NSError * _Nullable * _Nullable)error; - (BOOL)eraseSimulatorWithUDID:(NSString *)udid error:(NSError * _Nullable * _Nullable)error; - (BOOL)installAppAtPath:(NSString *)appPath simulatorUDID:(NSString *)udid error:(NSError * _Nullable * _Nullable)error; - (BOOL)uninstallBundleID:(NSString *)bundleID simulatorUDID:(NSString *)udid error:(NSError * _Nullable * _Nullable)error; diff --git a/cli/XCWSimctl.m b/cli/XCWSimctl.m index ca37e1e6..14122303 100644 --- a/cli/XCWSimctl.m +++ b/cli/XCWSimctl.m @@ -2,10 +2,13 @@ #import +#import "XCWChromeRenderer.h" #import "XCWPrivateSimulatorBooter.h" #import "XCWProcessRunner.h" #import +#import +#import #import #import @@ -22,6 +25,10 @@ + (nullable XCWProcessResult *)runSimctl:(NSArray *)arguments + (nullable XCWProcessResult *)runSimctl:(NSArray *)arguments timeoutSec:(NSTimeInterval)timeoutSec error:(NSError * _Nullable __autoreleasing *)error; ++ (nullable XCWProcessResult *)runSimctl:(NSArray *)arguments + timeoutSec:(NSTimeInterval)timeoutSec + timeoutSignal:(int)timeoutSignal + error:(NSError * _Nullable __autoreleasing *)error; + (nullable NSDictionary *)listJSONPayloadWithError:(NSError * _Nullable __autoreleasing *)error; + (NSError *)errorWithDescription:(NSString *)description code:(NSInteger)code; - (BOOL)installAppBundleAtPath:(NSString *)appPath simulatorUDID:(NSString *)udid error:(NSError * _Nullable __autoreleasing *)error; @@ -623,6 +630,12 @@ - (BOOL)launchBundleID:(NSString *)bundleID simulatorUDID:(NSString *)udid error } - (nullable NSData *)screenshotPNGForSimulatorUDID:(NSString *)udid error:(NSError * _Nullable __autoreleasing *)error { + return [self screenshotPNGForSimulatorUDID:udid includeBezel:NO error:error]; +} + +- (nullable NSData *)screenshotPNGForSimulatorUDID:(NSString *)udid + includeBezel:(BOOL)includeBezel + error:(NSError * _Nullable __autoreleasing *)error { NSString *filename = [NSString stringWithFormat:@"simdeck-%@.png", NSUUID.UUID.UUIDString]; NSString *path = [NSTemporaryDirectory() stringByAppendingPathComponent:filename]; XCWProcessResult *result = [self.class runSimctl:@[@"io", udid, @"screenshot", @"--type=png", path] error:error]; @@ -633,7 +646,17 @@ - (nullable NSData *)screenshotPNGForSimulatorUDID:(NSString *)udid error:(NSErr NSData *data = [NSData dataWithContentsOfFile:path options:0 error:error]; [[NSFileManager defaultManager] removeItemAtPath:path error:nil]; if (data.length > 0) { - return data; + if (!includeBezel) { + return data; + } + NSDictionary *simulator = [self simulatorWithUDID:udid error:error]; + if (simulator == nil) { + return nil; + } + NSString *deviceName = simulator[@"deviceTypeName"] ?: simulator[@"name"] ?: @""; + return [XCWChromeRenderer screenshotPNGDataForDeviceName:deviceName + screenPNGData:data + error:error]; } if (error != NULL && *error == nil) { *error = [self.class errorWithDescription:@"Simulator screenshot command produced an empty PNG." code:13]; @@ -647,6 +670,54 @@ - (nullable NSData *)screenshotPNGForSimulatorUDID:(NSString *)udid error:(NSErr return nil; } +- (nullable NSData *)screenRecordingMP4ForSimulatorUDID:(NSString *)udid + durationSeconds:(NSTimeInterval)durationSeconds + error:(NSError * _Nullable __autoreleasing *)error { + if (!isfinite(durationSeconds) || durationSeconds <= 0.0) { + if (error != NULL) { + *error = [self.class errorWithDescription:@"Screen recording duration must be finite and greater than zero." code:33]; + } + return nil; + } + + NSString *filename = [NSString stringWithFormat:@"simdeck-%@.mp4", NSUUID.UUID.UUIDString]; + NSString *path = [NSTemporaryDirectory() stringByAppendingPathComponent:filename]; + XCWProcessResult *result = [self.class runSimctl:@[ + @"io", + udid, + @"recordVideo", + @"--codec=h264", + @"--force", + path, + ] + timeoutSec:durationSeconds + timeoutSignal:SIGINT + error:error]; + if (result == nil) { + [[NSFileManager defaultManager] removeItemAtPath:path error:nil]; + return nil; + } + + BOOL expectedStop = result.terminationStatus == 0 || result.terminationStatus == 124 || result.terminationStatus == 130; + if (expectedStop) { + NSData *data = [NSData dataWithContentsOfFile:path options:0 error:error]; + [[NSFileManager defaultManager] removeItemAtPath:path error:nil]; + if (data.length > 0) { + return data; + } + if (error != NULL && *error == nil) { + *error = [self.class errorWithDescription:@"Simulator screen recording command produced an empty MP4." code:34]; + } + return nil; + } + + [[NSFileManager defaultManager] removeItemAtPath:path error:nil]; + if (error != NULL) { + *error = [self.class errorWithDescription:result.stderrString.length > 0 ? result.stderrString : @"Unable to record simulator screen." code:34]; + } + return nil; +} + - (BOOL)eraseSimulatorWithUDID:(NSString *)udid error:(NSError * _Nullable __autoreleasing *)error { XCWProcessResult *result = [self.class runSimctl:@[@"erase", udid] error:error]; if (result == nil) { @@ -863,10 +934,18 @@ + (nullable NSDictionary *)listJSONPayloadWithError:(NSError * _Nullable __autor + (nullable XCWProcessResult *)runSimctl:(NSArray *)arguments timeoutSec:(NSTimeInterval)timeoutSec error:(NSError * _Nullable __autoreleasing *)error { + return [self runSimctl:arguments timeoutSec:timeoutSec timeoutSignal:SIGTERM error:error]; +} + ++ (nullable XCWProcessResult *)runSimctl:(NSArray *)arguments + timeoutSec:(NSTimeInterval)timeoutSec + timeoutSignal:(int)timeoutSignal + error:(NSError * _Nullable __autoreleasing *)error { return [XCWProcessRunner runLaunchPath:@"/usr/bin/xcrun" arguments:[@[@"simctl"] arrayByAddingObjectsFromArray:arguments] inputData:nil timeoutSec:timeoutSec + timeoutSignal:timeoutSignal error:error]; } diff --git a/cli/native/XCWNativeBridge.h b/cli/native/XCWNativeBridge.h index d8a35240..587e7296 100644 --- a/cli/native/XCWNativeBridge.h +++ b/cli/native/XCWNativeBridge.h @@ -51,7 +51,8 @@ char * _Nullable xcw_native_get_chrome_profile(const char * _Nonnull udid, char xcw_native_owned_bytes xcw_native_render_chrome_png(const char * _Nonnull udid, bool include_buttons, char * _Nullable * _Nullable error_message); xcw_native_owned_bytes xcw_native_render_chrome_button_png(const char * _Nonnull udid, const char * _Nonnull button_name, bool pressed, char * _Nullable * _Nullable error_message); xcw_native_owned_bytes xcw_native_render_screen_mask_png(const char * _Nonnull udid, char * _Nullable * _Nullable error_message); -xcw_native_owned_bytes xcw_native_screenshot_png(const char * _Nonnull udid, char * _Nullable * _Nullable error_message); +xcw_native_owned_bytes xcw_native_screenshot_png(const char * _Nonnull udid, bool include_bezel, char * _Nullable * _Nullable error_message); +xcw_native_owned_bytes xcw_native_screen_recording_mp4(const char * _Nonnull udid, double duration_seconds, char * _Nullable * _Nullable error_message); char * _Nullable xcw_native_recent_logs(const char * _Nonnull udid, double seconds, size_t limit, char * _Nullable * _Nullable error_message); char * _Nullable xcw_native_accessibility_snapshot(const char * _Nonnull udid, bool has_point, double x, double y, size_t max_depth, char * _Nullable * _Nullable error_message); bool xcw_native_send_touch(const char * _Nonnull udid, double x, double y, const char * _Nonnull phase, char * _Nullable * _Nullable error_message); diff --git a/cli/native/XCWNativeBridge.m b/cli/native/XCWNativeBridge.m index 1193a9e2..48840c44 100644 --- a/cli/native/XCWNativeBridge.m +++ b/cli/native/XCWNativeBridge.m @@ -532,11 +532,13 @@ xcw_native_owned_bytes xcw_native_render_screen_mask_png(const char *udid, char } } -xcw_native_owned_bytes xcw_native_screenshot_png(const char *udid, char **error_message) { +xcw_native_owned_bytes xcw_native_screenshot_png(const char *udid, bool include_bezel, char **error_message) { @autoreleasepool { XCWSimctl *simctl = [[XCWSimctl alloc] init]; NSError *error = nil; - NSData *png = [simctl screenshotPNGForSimulatorUDID:XCWStringFromCString(udid) error:&error]; + NSData *png = [simctl screenshotPNGForSimulatorUDID:XCWStringFromCString(udid) + includeBezel:include_bezel + error:&error]; if (png == nil) { XCWSetErrorMessage(error_message, error); return (xcw_native_owned_bytes){0}; @@ -545,6 +547,21 @@ xcw_native_owned_bytes xcw_native_screenshot_png(const char *udid, char **error_ } } +xcw_native_owned_bytes xcw_native_screen_recording_mp4(const char *udid, double duration_seconds, char **error_message) { + @autoreleasepool { + XCWSimctl *simctl = [[XCWSimctl alloc] init]; + NSError *error = nil; + NSData *mp4 = [simctl screenRecordingMP4ForSimulatorUDID:XCWStringFromCString(udid) + durationSeconds:duration_seconds + error:&error]; + if (mp4 == nil) { + XCWSetErrorMessage(error_message, error); + return (xcw_native_owned_bytes){0}; + } + return XCWOwnedBytesFromData(mp4); + } +} + char *xcw_native_recent_logs(const char *udid, double seconds, size_t limit, char **error_message) { @autoreleasepool { XCWSimctl *simctl = [[XCWSimctl alloc] init]; diff --git a/client/src/api/controls.ts b/client/src/api/controls.ts index 40471bbe..7f7c4dcd 100644 --- a/client/src/api/controls.ts +++ b/client/src/api/controls.ts @@ -1,4 +1,4 @@ -import { accessTokenFromLocation, apiRequest } from "./client"; +import { accessTokenFromLocation, apiHeaders, apiRequest } from "./client"; import { apiUrl } from "./config"; import type { ButtonPayload, @@ -135,3 +135,51 @@ export function rotateLeft(udid: string) { export function rotateRight(udid: string) { return postSimulatorAction(udid, "rotate-right"); } + +async function fetchSimulatorBlob( + path: string, + options: RequestInit = {}, +): Promise { + const { headers, ...rest } = options; + const response = await fetch(apiUrl(path), { + ...rest, + headers: apiHeaders(headers), + }); + if (!response.ok) { + const contentType = response.headers.get("content-type") ?? ""; + if (contentType.includes("application/json")) { + const body = (await response.json()) as { error?: string }; + throw new Error( + body.error ?? `Request failed with status ${response.status}`, + ); + } + throw new Error( + (await response.text()) || + `Request failed with status ${response.status}`, + ); + } + return response.blob(); +} + +export function captureSimulatorScreenshot( + udid: string, + options: { withBezel?: boolean } = {}, +): Promise { + const params = options.withBezel ? "?bezel=true" : ""; + return fetchSimulatorBlob( + `/api/simulators/${encodeURIComponent(udid)}/screenshot.png${params}`, + ); +} + +export function recordSimulatorScreen( + udid: string, + seconds = 5, +): Promise { + return fetchSimulatorBlob( + `/api/simulators/${encodeURIComponent(udid)}/screen-recording`, + { + body: JSON.stringify({ seconds }), + method: "POST", + }, + ); +} diff --git a/client/src/app/AppShell.tsx b/client/src/app/AppShell.tsx index 5f4aff98..46032837 100644 --- a/client/src/app/AppShell.tsx +++ b/client/src/app/AppShell.tsx @@ -17,12 +17,14 @@ import { import { apiUrl, configureSimDeckClient } from "../api/config"; import { bootSimulator, + captureSimulatorScreenshot, dismissKeyboard, launchSimulatorBundle, openAppSwitcher, openSimulatorUrl, pressHome, pressSimulatorButton, + recordSimulatorScreen, rotateDigitalCrown, rotateRight, simulatorControlSocketUrl, @@ -268,6 +270,25 @@ function writeStreamTransportQueryParam(transport: StreamTransport) { ); } +function downloadBlob(blob: Blob, fileName: string) { + const url = URL.createObjectURL(blob); + const link = document.createElement("a"); + link.href = url; + link.download = fileName; + document.body.appendChild(link); + link.click(); + link.remove(); + URL.revokeObjectURL(url); +} + +function captureFileBaseName( + simulator: SimulatorMetadata, + artifact: "Recording" | "Screenshot", +): string { + const safeName = simulator.name.replace(/[^A-Za-z0-9._-]+/g, "-"); + return `SimDeck ${artifact} - ${safeName || simulator.udid}`; +} + function simulatorDisplaySize( simulator: SimulatorMetadata | null, ): Size | null { @@ -1746,6 +1767,48 @@ export function AppShell({ } } + async function downloadSimulatorScreenshot(withBezel: boolean) { + if (!selectedSimulator) { + return; + } + setLocalError(""); + try { + const blob = await captureSimulatorScreenshot(selectedSimulator.udid, { + withBezel, + }); + downloadBlob( + blob, + `${captureFileBaseName(selectedSimulator, "Screenshot")}${withBezel ? " Bezel" : ""}.png`, + ); + } catch (captureError) { + setLocalError( + captureError instanceof Error + ? captureError.message + : "Capture failed.", + ); + } + } + + async function downloadSimulatorRecording() { + if (!selectedSimulator) { + return; + } + setLocalError(""); + try { + const blob = await recordSimulatorScreen(selectedSimulator.udid, 5); + downloadBlob( + blob, + `${captureFileBaseName(selectedSimulator, "Recording")}.mp4`, + ); + } catch (captureError) { + setLocalError( + captureError instanceof Error + ? captureError.message + : "Recording failed.", + ); + } + } + function selectedStateFromSimulator( simulator: SimulatorMetadata, current: SimulatorStateResponse | null, @@ -2478,6 +2541,12 @@ export function AppShell({ bootSimulator(udid), ); }} + onCaptureScreenshot={() => { + void downloadSimulatorScreenshot(false); + }} + onCaptureScreenshotWithBezel={() => { + void downloadSimulatorScreenshot(true); + }} onChangeSearch={setSearch} onDismissKeyboard={() => { if (!selectedSimulator) { @@ -2546,6 +2615,9 @@ export function AppShell({ setRotationQuarterTurns((current) => (current + 1) % 4); }, false); }} + onRecordScreen={() => { + void downloadSimulatorRecording(); + }} onStreamEncoderChange={updateStreamEncoder} onStreamFpsChange={updateStreamFps} onStreamQualityChange={updateStreamQuality} diff --git a/client/src/features/simulators/SimulatorMenu.tsx b/client/src/features/simulators/SimulatorMenu.tsx index fe739271..f7f7aacd 100644 --- a/client/src/features/simulators/SimulatorMenu.tsx +++ b/client/src/features/simulators/SimulatorMenu.tsx @@ -23,6 +23,8 @@ interface SimulatorMenuProps { menuOpen: boolean; menuRef: RefObject; onBoot: () => void; + onCaptureScreenshot: () => void; + onCaptureScreenshotWithBezel: () => void; onChangeSearch: (value: string) => void; onCloseMenu: () => void; onDismissKeyboard: () => void; @@ -33,6 +35,7 @@ interface SimulatorMenuProps { onOpenNewSimulator: () => void; onOpenUrlPrompt: () => void; onRotateRight: () => void; + onRecordScreen: () => void; onShutdown: () => void; onStreamEncoderChange: (encoder: StreamEncoder) => void; onStreamFpsChange: (fps: StreamFps) => void; @@ -62,6 +65,8 @@ export function SimulatorMenu({ menuOpen, menuRef, onBoot, + onCaptureScreenshot, + onCaptureScreenshotWithBezel, onChangeSearch, onCloseMenu, onDismissKeyboard, @@ -72,6 +77,7 @@ export function SimulatorMenu({ onOpenNewSimulator, onOpenUrlPrompt, onRotateRight, + onRecordScreen, onShutdown, onStreamEncoderChange, onStreamFpsChange, @@ -280,6 +286,33 @@ export function SimulatorMenu({ + + +