diff --git a/AGENTS.md b/AGENTS.md index e1a46833..a5026d2c 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -137,6 +137,8 @@ Useful direct commands: ./build/simdeck pasteboard set "hello" ./build/simdeck pasteboard get ./build/simdeck screenshot --output screen.png +./build/simdeck screenshot --with-bezel --output screen-bezel.png +./build/simdeck record --seconds 5 --output screen-recording.mp4 ./build/simdeck describe ./build/simdeck tap 120 240 ./build/simdeck tap --label "Continue" --wait-timeout-ms 5000 diff --git a/README.md b/README.md index 416270ef..fa9e63e6 100644 --- a/README.md +++ b/README.md @@ -130,6 +130,8 @@ simdeck toggle-appearance simdeck pasteboard set "hello" simdeck pasteboard get simdeck screenshot --output screen.png +simdeck screenshot --with-bezel --output screen-bezel.png +simdeck record --seconds 5 --output screen-recording.mp4 simdeck stream --frames 120 > stream.h264 simdeck describe simdeck describe --format agent --max-depth 4 @@ -206,6 +208,8 @@ try { await sim.tap(0.5, 0.5); await sim.waitFor({ label: "Continue" }); await sim.screenshot(); + await sim.screenshot({ withBezel: true }); + await sim.record({ seconds: 5 }); } finally { sim.close(); } diff --git a/cli/XCWChromeRenderer.h b/cli/XCWChromeRenderer.h index 64f7c513..2a30175e 100644 --- a/cli/XCWChromeRenderer.h +++ b/cli/XCWChromeRenderer.h @@ -15,6 +15,9 @@ NS_ASSUME_NONNULL_BEGIN error:(NSError * _Nullable * _Nullable)error; + (nullable NSData *)screenMaskPNGDataForDeviceName:(NSString *)deviceName error:(NSError * _Nullable * _Nullable)error; ++ (nullable NSData *)screenshotPNGDataForDeviceName:(NSString *)deviceName + screenPNGData:(NSData *)screenPNGData + error:(NSError * _Nullable * _Nullable)error; + (nullable NSDictionary *)profileForDeviceName:(NSString *)deviceName error:(NSError * _Nullable * _Nullable)error; diff --git a/cli/XCWChromeRenderer.m b/cli/XCWChromeRenderer.m index dea918ee..abd4feab 100644 --- a/cli/XCWChromeRenderer.m +++ b/cli/XCWChromeRenderer.m @@ -249,6 +249,111 @@ + (nullable NSData *)screenMaskPNGDataForDeviceName:(NSString *)deviceName return [self PNGDataForPDFAtPath:maskPath scale:1.0 error:error]; } ++ (nullable NSData *)screenshotPNGDataForDeviceName:(NSString *)deviceName + screenPNGData:(NSData *)screenPNGData + error:(NSError * _Nullable __autoreleasing *)error { + NSDictionary *profile = [self profileForDeviceName:deviceName error:error]; + if (profile == nil) { + return nil; + } + + NSData *chromePNGData = [self PNGDataForDeviceName:deviceName includeButtons:YES error:error]; + if (chromePNGData == nil) { + return nil; + } + + NSImage *screenImage = [[NSImage alloc] initWithData:screenPNGData]; + NSImage *chromeImage = [[NSImage alloc] initWithData:chromePNGData]; + if (screenImage == nil || chromeImage == nil) { + if (error != NULL) { + *error = [NSError errorWithDomain:XCWChromeRendererErrorDomain + code:15 + userInfo:@{ + NSLocalizedDescriptionKey: @"Unable to decode simulator screenshot or chrome PNG data.", + }]; + } + return nil; + } + + CGFloat scale = 3.0; + CGFloat totalWidth = [self numberValue:profile[@"totalWidth"]]; + CGFloat totalHeight = [self numberValue:profile[@"totalHeight"]]; + CGFloat screenX = [self numberValue:profile[@"screenX"]]; + CGFloat screenY = [self numberValue:profile[@"screenY"]]; + CGFloat screenWidth = [self numberValue:profile[@"screenWidth"]]; + CGFloat screenHeight = [self numberValue:profile[@"screenHeight"]]; + NSInteger pixelWidth = MAX((NSInteger)ceil(totalWidth * scale), 1); + NSInteger pixelHeight = MAX((NSInteger)ceil(totalHeight * scale), 1); + if (totalWidth <= 0.0 || totalHeight <= 0.0 || screenWidth <= 0.0 || screenHeight <= 0.0) { + if (error != NULL) { + *error = [NSError errorWithDomain:XCWChromeRendererErrorDomain + code:16 + userInfo:@{ + NSLocalizedDescriptionKey: @"Device chrome profile did not include usable screenshot geometry.", + }]; + } + return nil; + } + + NSBitmapImageRep *bitmap = [[NSBitmapImageRep alloc] initWithBitmapDataPlanes:NULL + pixelsWide:pixelWidth + pixelsHigh:pixelHeight + bitsPerSample:8 + samplesPerPixel:4 + hasAlpha:YES + isPlanar:NO + colorSpaceName:NSDeviceRGBColorSpace + bytesPerRow:0 + bitsPerPixel:32]; + if (bitmap == nil) { + if (error != NULL) { + *error = [NSError errorWithDomain:XCWChromeRendererErrorDomain + code:17 + userInfo:@{ + NSLocalizedDescriptionKey: @"Unable to create a bitmap for bezeled screenshot rendering.", + }]; + } + return nil; + } + + NSGraphicsContext *graphicsContext = [NSGraphicsContext graphicsContextWithBitmapImageRep:bitmap]; + [NSGraphicsContext saveGraphicsState]; + [NSGraphicsContext setCurrentContext:graphicsContext]; + graphicsContext.imageInterpolation = NSImageInterpolationHigh; + NSRect outputRect = NSMakeRect(0.0, 0.0, pixelWidth, pixelHeight); + [[NSColor clearColor] setFill]; + NSRectFillUsingOperation(outputRect, NSCompositingOperationClear); + + NSRect screenRect = NSMakeRect(screenX * scale, + pixelHeight - ((screenY + screenHeight) * scale), + screenWidth * scale, + screenHeight * scale); + NSDictionary *hints = @{ NSImageHintInterpolation: @(NSImageInterpolationHigh) }; + [screenImage drawInRect:screenRect + fromRect:NSZeroRect + operation:NSCompositingOperationSourceOver + fraction:1.0 + respectFlipped:NO + hints:hints]; + [chromeImage drawInRect:outputRect + fromRect:NSZeroRect + operation:NSCompositingOperationSourceOver + fraction:1.0 + respectFlipped:NO + hints:hints]; + [NSGraphicsContext restoreGraphicsState]; + + NSData *pngData = [bitmap representationUsingType:NSBitmapImageFileTypePNG properties:@{}]; + if (pngData.length == 0 && error != NULL) { + *error = [NSError errorWithDomain:XCWChromeRendererErrorDomain + code:18 + userInfo:@{ + NSLocalizedDescriptionKey: @"Unable to encode bezeled simulator screenshot PNG.", + }]; + } + return pngData.length > 0 ? pngData : nil; +} + + (nullable NSDictionary *)profileForChromeInfo:(NSDictionary *)chromeInfo error:(NSError * _Nullable __autoreleasing *)error { NSDictionary *plist = chromeInfo[@"plist"]; diff --git a/cli/XCWProcessRunner.h b/cli/XCWProcessRunner.h index 70626ab2..f76b8580 100644 --- a/cli/XCWProcessRunner.h +++ b/cli/XCWProcessRunner.h @@ -31,6 +31,13 @@ NS_ASSUME_NONNULL_BEGIN timeoutSec:(NSTimeInterval)timeoutSec error:(NSError * _Nullable * _Nullable)error; ++ (XCWProcessResult *)runLaunchPath:(NSString *)launchPath + arguments:(NSArray *)arguments + inputData:(nullable NSData *)inputData + timeoutSec:(NSTimeInterval)timeoutSec + timeoutSignal:(int)timeoutSignal + error:(NSError * _Nullable * _Nullable)error; + @end NS_ASSUME_NONNULL_END diff --git a/cli/XCWProcessRunner.m b/cli/XCWProcessRunner.m index fbf61028..5e4c2fde 100644 --- a/cli/XCWProcessRunner.m +++ b/cli/XCWProcessRunner.m @@ -119,6 +119,20 @@ + (XCWProcessResult *)runLaunchPath:(NSString *)launchPath inputData:(NSData *)inputData timeoutSec:(NSTimeInterval)timeoutSec error:(NSError * _Nullable __autoreleasing *)error { + return [self runLaunchPath:launchPath + arguments:arguments + inputData:inputData + timeoutSec:timeoutSec + timeoutSignal:SIGTERM + error:error]; +} + ++ (XCWProcessResult *)runLaunchPath:(NSString *)launchPath + arguments:(NSArray *)arguments + inputData:(NSData *)inputData + timeoutSec:(NSTimeInterval)timeoutSec + timeoutSignal:(int)timeoutSignal + error:(NSError * _Nullable __autoreleasing *)error { int stdoutFD = -1; int stderrFD = -1; int stdinPipe[2] = { -1, -1 }; @@ -244,8 +258,10 @@ + (XCWProcessResult *)runLaunchPath:(NSString *)launchPath } if (hasTimeout && [deadline timeIntervalSinceNow] <= 0) { timedOut = YES; - kill(pid, SIGTERM); - NSDate *killDeadline = [NSDate dateWithTimeIntervalSinceNow:2.0]; + int signalToSend = timeoutSignal > 0 ? timeoutSignal : SIGTERM; + kill(pid, signalToSend); + NSTimeInterval graceSeconds = signalToSend == SIGINT ? 10.0 : 2.0; + NSDate *killDeadline = [NSDate dateWithTimeIntervalSinceNow:graceSeconds]; do { waitResult = waitpid(pid, &waitStatus, WNOHANG); if (waitResult == pid || (waitResult < 0 && errno != EINTR)) { diff --git a/cli/XCWSimctl.h b/cli/XCWSimctl.h index 430821d3..d8a0932b 100644 --- a/cli/XCWSimctl.h +++ b/cli/XCWSimctl.h @@ -19,6 +19,12 @@ NS_ASSUME_NONNULL_BEGIN - (BOOL)openURL:(NSString *)urlString simulatorUDID:(NSString *)udid error:(NSError * _Nullable * _Nullable)error; - (BOOL)launchBundleID:(NSString *)bundleID simulatorUDID:(NSString *)udid error:(NSError * _Nullable * _Nullable)error; - (nullable NSData *)screenshotPNGForSimulatorUDID:(NSString *)udid error:(NSError * _Nullable * _Nullable)error; +- (nullable NSData *)screenshotPNGForSimulatorUDID:(NSString *)udid + includeBezel:(BOOL)includeBezel + error:(NSError * _Nullable * _Nullable)error; +- (nullable NSData *)screenRecordingMP4ForSimulatorUDID:(NSString *)udid + durationSeconds:(NSTimeInterval)durationSeconds + error:(NSError * _Nullable * _Nullable)error; - (BOOL)eraseSimulatorWithUDID:(NSString *)udid error:(NSError * _Nullable * _Nullable)error; - (BOOL)installAppAtPath:(NSString *)appPath simulatorUDID:(NSString *)udid error:(NSError * _Nullable * _Nullable)error; - (BOOL)uninstallBundleID:(NSString *)bundleID simulatorUDID:(NSString *)udid error:(NSError * _Nullable * _Nullable)error; diff --git a/cli/XCWSimctl.m b/cli/XCWSimctl.m index ca37e1e6..efa8e6ea 100644 --- a/cli/XCWSimctl.m +++ b/cli/XCWSimctl.m @@ -2,10 +2,14 @@ #import +#import "XCWChromeRenderer.h" #import "XCWPrivateSimulatorBooter.h" #import "XCWProcessRunner.h" +#import #import +#import +#import #import #import @@ -22,6 +26,10 @@ + (nullable XCWProcessResult *)runSimctl:(NSArray *)arguments + (nullable XCWProcessResult *)runSimctl:(NSArray *)arguments timeoutSec:(NSTimeInterval)timeoutSec error:(NSError * _Nullable __autoreleasing *)error; ++ (nullable XCWProcessResult *)runSimctl:(NSArray *)arguments + timeoutSec:(NSTimeInterval)timeoutSec + timeoutSignal:(int)timeoutSignal + error:(NSError * _Nullable __autoreleasing *)error; + (nullable NSDictionary *)listJSONPayloadWithError:(NSError * _Nullable __autoreleasing *)error; + (NSError *)errorWithDescription:(NSString *)description code:(NSInteger)code; - (BOOL)installAppBundleAtPath:(NSString *)appPath simulatorUDID:(NSString *)udid error:(NSError * _Nullable __autoreleasing *)error; @@ -46,10 +54,47 @@ - (BOOL)installIPAAtPath:(NSString *)ipaPath simulatorUDID:(NSString *)udid erro return [value isKindOfClass:[NSString class]] ? value : @""; } +static NSString *XCWTrimmedString(NSString *value) { + return [value stringByTrimmingCharactersInSet:NSCharacterSet.whitespaceAndNewlineCharacterSet]; +} + static NSNumber *XCWNumberValue(id value) { return [value isKindOfClass:[NSNumber class]] ? value : nil; } +static NSString *XCWStringFromData(NSData *data) { + return [[NSString alloc] initWithData:data encoding:NSUTF8StringEncoding] ?: @""; +} + +static NSData *XCWDataSnapshot(NSMutableData *data) { + @synchronized(data) { + return [data copy]; + } +} + +static void XCWAppendAvailableData(NSFileHandle *handle, NSMutableData *destination) { + @try { + NSData *chunk = [handle availableData]; + if (chunk.length > 0) { + @synchronized(destination) { + [destination appendData:chunk]; + } + } + } @catch (NSException *exception) { + } +} + +static BOOL XCWWaitForTaskExit(NSTask *task, NSTimeInterval timeoutSeconds) { + NSDate *deadline = [NSDate dateWithTimeIntervalSinceNow:timeoutSeconds]; + while (task.running && [deadline timeIntervalSinceNow] > 0) { + usleep(10 * 1000); + } + if (!task.running) { + return YES; + } + return NO; +} + static NSString * _Nullable XCWCreateTemporaryDirectory(NSString *prefix, NSError * _Nullable __autoreleasing *error) { NSString *templatePath = [NSTemporaryDirectory() stringByAppendingPathComponent:[NSString stringWithFormat:@"%@-XXXXXX", prefix]]; char *directoryTemplate = strdup(templatePath.fileSystemRepresentation); @@ -623,6 +668,12 @@ - (BOOL)launchBundleID:(NSString *)bundleID simulatorUDID:(NSString *)udid error } - (nullable NSData *)screenshotPNGForSimulatorUDID:(NSString *)udid error:(NSError * _Nullable __autoreleasing *)error { + return [self screenshotPNGForSimulatorUDID:udid includeBezel:NO error:error]; +} + +- (nullable NSData *)screenshotPNGForSimulatorUDID:(NSString *)udid + includeBezel:(BOOL)includeBezel + error:(NSError * _Nullable __autoreleasing *)error { NSString *filename = [NSString stringWithFormat:@"simdeck-%@.png", NSUUID.UUID.UUIDString]; NSString *path = [NSTemporaryDirectory() stringByAppendingPathComponent:filename]; XCWProcessResult *result = [self.class runSimctl:@[@"io", udid, @"screenshot", @"--type=png", path] error:error]; @@ -633,7 +684,17 @@ - (nullable NSData *)screenshotPNGForSimulatorUDID:(NSString *)udid error:(NSErr NSData *data = [NSData dataWithContentsOfFile:path options:0 error:error]; [[NSFileManager defaultManager] removeItemAtPath:path error:nil]; if (data.length > 0) { - return data; + if (!includeBezel) { + return data; + } + NSDictionary *simulator = [self simulatorWithUDID:udid error:error]; + if (simulator == nil) { + return nil; + } + NSString *deviceName = simulator[@"deviceTypeName"] ?: simulator[@"name"] ?: @""; + return [XCWChromeRenderer screenshotPNGDataForDeviceName:deviceName + screenPNGData:data + error:error]; } if (error != NULL && *error == nil) { *error = [self.class errorWithDescription:@"Simulator screenshot command produced an empty PNG." code:13]; @@ -647,6 +708,157 @@ - (nullable NSData *)screenshotPNGForSimulatorUDID:(NSString *)udid error:(NSErr return nil; } +- (nullable NSData *)screenRecordingMP4ForSimulatorUDID:(NSString *)udid + durationSeconds:(NSTimeInterval)durationSeconds + error:(NSError * _Nullable __autoreleasing *)error { + if (!isfinite(durationSeconds) || durationSeconds <= 0.0) { + if (error != NULL) { + *error = [self.class errorWithDescription:@"Screen recording duration must be finite and greater than zero." code:33]; + } + return nil; + } + + NSString *filename = [NSString stringWithFormat:@"simdeck-%@.mp4", NSUUID.UUID.UUIDString]; + NSString *path = [NSTemporaryDirectory() stringByAppendingPathComponent:filename]; + XCWProcessResult *resolvedSimctl = [XCWProcessRunner runLaunchPath:@"/usr/bin/xcrun" + arguments:@[@"-f", @"simctl"] + inputData:nil + error:error]; + if (resolvedSimctl == nil) { + [[NSFileManager defaultManager] removeItemAtPath:path error:nil]; + return nil; + } + NSString *simctlPath = XCWTrimmedString(XCWStringFromData(resolvedSimctl.stdoutData)); + if (resolvedSimctl.terminationStatus != 0 || simctlPath.length == 0) { + [[NSFileManager defaultManager] removeItemAtPath:path error:nil]; + if (error != NULL) { + NSString *message = resolvedSimctl.stderrString.length > 0 ? resolvedSimctl.stderrString : @"Unable to locate simctl."; + *error = [self.class errorWithDescription:message code:34]; + } + return nil; + } + + NSTask *task = [[NSTask alloc] init]; + task.launchPath = simctlPath; + task.arguments = @[@"io", udid, @"recordVideo", @"--codec=h264", @"--force", path]; + task.standardInput = NSFileHandle.fileHandleWithNullDevice; + NSPipe *stdoutPipe = [NSPipe pipe]; + NSPipe *stderrPipe = [NSPipe pipe]; + task.standardOutput = stdoutPipe; + task.standardError = stderrPipe; + + NSMutableData *stdoutData = [NSMutableData data]; + NSMutableData *stderrData = [NSMutableData data]; + NSFileHandle *stdoutHandle = stdoutPipe.fileHandleForReading; + NSFileHandle *stderrHandle = stderrPipe.fileHandleForReading; + dispatch_semaphore_t recordingStartedSemaphore = dispatch_semaphore_create(0); + __block BOOL recordingStarted = NO; + + stdoutHandle.readabilityHandler = ^(NSFileHandle *handle) { + XCWAppendAvailableData(handle, stdoutData); + }; + stderrHandle.readabilityHandler = ^(NSFileHandle *handle) { + @try { + NSData *chunk = [handle availableData]; + if (chunk.length == 0) { + return; + } + BOOL shouldSignal = NO; + @synchronized(stderrData) { + [stderrData appendData:chunk]; + NSString *text = XCWStringFromData(stderrData); + if (!recordingStarted && [text rangeOfString:@"Recording started"].location != NSNotFound) { + recordingStarted = YES; + shouldSignal = YES; + } + } + if (shouldSignal) { + dispatch_semaphore_signal(recordingStartedSemaphore); + } + } @catch (NSException *exception) { + } + }; + + NSError *launchError = nil; + if (![task launchAndReturnError:&launchError]) { + stdoutHandle.readabilityHandler = nil; + stderrHandle.readabilityHandler = nil; + [[NSFileManager defaultManager] removeItemAtPath:path error:nil]; + if (error != NULL) { + *error = launchError ?: [self.class errorWithDescription:@"Unable to launch simulator screen recording." code:34]; + } + return nil; + } + + NSTimeInterval startTimeout = MAX(10.0, MIN(30.0, durationSeconds + 10.0)); + NSDate *startDeadline = [NSDate dateWithTimeIntervalSinceNow:startTimeout]; + BOOL didStart = NO; + while ([startDeadline timeIntervalSinceNow] > 0) { + if (dispatch_semaphore_wait(recordingStartedSemaphore, DISPATCH_TIME_NOW) == 0) { + didStart = YES; + break; + } + if (!task.running) { + break; + } + usleep(10 * 1000); + } + + if (didStart) { + NSDate *stopAt = [NSDate dateWithTimeIntervalSinceNow:durationSeconds]; + while (task.running && [stopAt timeIntervalSinceNow] > 0) { + usleep(10 * 1000); + } + } + if (task.running) { + [task interrupt]; + } + if (!XCWWaitForTaskExit(task, didStart ? 10.0 : 2.0) && task.running) { + [task terminate]; + if (!XCWWaitForTaskExit(task, 2.0) && task.running) { + kill(task.processIdentifier, SIGKILL); + XCWWaitForTaskExit(task, 2.0); + } + } + + stdoutHandle.readabilityHandler = nil; + stderrHandle.readabilityHandler = nil; + if (!task.running) { + XCWAppendAvailableData(stdoutHandle, stdoutData); + XCWAppendAvailableData(stderrHandle, stderrData); + } + + NSError *readError = nil; + NSData *data = nil; + NSDate *fileDeadline = [NSDate dateWithTimeIntervalSinceNow:2.0]; + do { + data = [NSData dataWithContentsOfFile:path options:0 error:&readError]; + if (data.length > 0) { + break; + } + usleep(50 * 1000); + } while ([fileDeadline timeIntervalSinceNow] > 0); + [[NSFileManager defaultManager] removeItemAtPath:path error:nil]; + if (data.length > 0) { + return data; + } + + if (error != NULL) { + NSString *stderrString = XCWStringFromData(XCWDataSnapshot(stderrData)); + NSString *stdoutString = XCWStringFromData(XCWDataSnapshot(stdoutData)); + NSString *details = stderrString.length > 0 ? stderrString : stdoutString; + if (!didStart && details.length == 0) { + details = @"Simulator screen recording did not start before the timeout."; + } else if (details.length == 0 && readError.localizedDescription.length > 0) { + details = readError.localizedDescription; + } else if (details.length == 0) { + details = @"Simulator screen recording command produced an empty MP4."; + } + *error = [self.class errorWithDescription:details code:34]; + } + return nil; +} + - (BOOL)eraseSimulatorWithUDID:(NSString *)udid error:(NSError * _Nullable __autoreleasing *)error { XCWProcessResult *result = [self.class runSimctl:@[@"erase", udid] error:error]; if (result == nil) { @@ -863,10 +1075,18 @@ + (nullable NSDictionary *)listJSONPayloadWithError:(NSError * _Nullable __autor + (nullable XCWProcessResult *)runSimctl:(NSArray *)arguments timeoutSec:(NSTimeInterval)timeoutSec error:(NSError * _Nullable __autoreleasing *)error { + return [self runSimctl:arguments timeoutSec:timeoutSec timeoutSignal:SIGTERM error:error]; +} + ++ (nullable XCWProcessResult *)runSimctl:(NSArray *)arguments + timeoutSec:(NSTimeInterval)timeoutSec + timeoutSignal:(int)timeoutSignal + error:(NSError * _Nullable __autoreleasing *)error { return [XCWProcessRunner runLaunchPath:@"/usr/bin/xcrun" arguments:[@[@"simctl"] arrayByAddingObjectsFromArray:arguments] inputData:nil timeoutSec:timeoutSec + timeoutSignal:timeoutSignal error:error]; } diff --git a/cli/native/XCWNativeBridge.h b/cli/native/XCWNativeBridge.h index d8a35240..587e7296 100644 --- a/cli/native/XCWNativeBridge.h +++ b/cli/native/XCWNativeBridge.h @@ -51,7 +51,8 @@ char * _Nullable xcw_native_get_chrome_profile(const char * _Nonnull udid, char xcw_native_owned_bytes xcw_native_render_chrome_png(const char * _Nonnull udid, bool include_buttons, char * _Nullable * _Nullable error_message); xcw_native_owned_bytes xcw_native_render_chrome_button_png(const char * _Nonnull udid, const char * _Nonnull button_name, bool pressed, char * _Nullable * _Nullable error_message); xcw_native_owned_bytes xcw_native_render_screen_mask_png(const char * _Nonnull udid, char * _Nullable * _Nullable error_message); -xcw_native_owned_bytes xcw_native_screenshot_png(const char * _Nonnull udid, char * _Nullable * _Nullable error_message); +xcw_native_owned_bytes xcw_native_screenshot_png(const char * _Nonnull udid, bool include_bezel, char * _Nullable * _Nullable error_message); +xcw_native_owned_bytes xcw_native_screen_recording_mp4(const char * _Nonnull udid, double duration_seconds, char * _Nullable * _Nullable error_message); char * _Nullable xcw_native_recent_logs(const char * _Nonnull udid, double seconds, size_t limit, char * _Nullable * _Nullable error_message); char * _Nullable xcw_native_accessibility_snapshot(const char * _Nonnull udid, bool has_point, double x, double y, size_t max_depth, char * _Nullable * _Nullable error_message); bool xcw_native_send_touch(const char * _Nonnull udid, double x, double y, const char * _Nonnull phase, char * _Nullable * _Nullable error_message); diff --git a/cli/native/XCWNativeBridge.m b/cli/native/XCWNativeBridge.m index 1193a9e2..48840c44 100644 --- a/cli/native/XCWNativeBridge.m +++ b/cli/native/XCWNativeBridge.m @@ -532,11 +532,13 @@ xcw_native_owned_bytes xcw_native_render_screen_mask_png(const char *udid, char } } -xcw_native_owned_bytes xcw_native_screenshot_png(const char *udid, char **error_message) { +xcw_native_owned_bytes xcw_native_screenshot_png(const char *udid, bool include_bezel, char **error_message) { @autoreleasepool { XCWSimctl *simctl = [[XCWSimctl alloc] init]; NSError *error = nil; - NSData *png = [simctl screenshotPNGForSimulatorUDID:XCWStringFromCString(udid) error:&error]; + NSData *png = [simctl screenshotPNGForSimulatorUDID:XCWStringFromCString(udid) + includeBezel:include_bezel + error:&error]; if (png == nil) { XCWSetErrorMessage(error_message, error); return (xcw_native_owned_bytes){0}; @@ -545,6 +547,21 @@ xcw_native_owned_bytes xcw_native_screenshot_png(const char *udid, char **error_ } } +xcw_native_owned_bytes xcw_native_screen_recording_mp4(const char *udid, double duration_seconds, char **error_message) { + @autoreleasepool { + XCWSimctl *simctl = [[XCWSimctl alloc] init]; + NSError *error = nil; + NSData *mp4 = [simctl screenRecordingMP4ForSimulatorUDID:XCWStringFromCString(udid) + durationSeconds:duration_seconds + error:&error]; + if (mp4 == nil) { + XCWSetErrorMessage(error_message, error); + return (xcw_native_owned_bytes){0}; + } + return XCWOwnedBytesFromData(mp4); + } +} + char *xcw_native_recent_logs(const char *udid, double seconds, size_t limit, char **error_message) { @autoreleasepool { XCWSimctl *simctl = [[XCWSimctl alloc] init]; diff --git a/client/src/api/controls.ts b/client/src/api/controls.ts index 40471bbe..7f7c4dcd 100644 --- a/client/src/api/controls.ts +++ b/client/src/api/controls.ts @@ -1,4 +1,4 @@ -import { accessTokenFromLocation, apiRequest } from "./client"; +import { accessTokenFromLocation, apiHeaders, apiRequest } from "./client"; import { apiUrl } from "./config"; import type { ButtonPayload, @@ -135,3 +135,51 @@ export function rotateLeft(udid: string) { export function rotateRight(udid: string) { return postSimulatorAction(udid, "rotate-right"); } + +async function fetchSimulatorBlob( + path: string, + options: RequestInit = {}, +): Promise { + const { headers, ...rest } = options; + const response = await fetch(apiUrl(path), { + ...rest, + headers: apiHeaders(headers), + }); + if (!response.ok) { + const contentType = response.headers.get("content-type") ?? ""; + if (contentType.includes("application/json")) { + const body = (await response.json()) as { error?: string }; + throw new Error( + body.error ?? `Request failed with status ${response.status}`, + ); + } + throw new Error( + (await response.text()) || + `Request failed with status ${response.status}`, + ); + } + return response.blob(); +} + +export function captureSimulatorScreenshot( + udid: string, + options: { withBezel?: boolean } = {}, +): Promise { + const params = options.withBezel ? "?bezel=true" : ""; + return fetchSimulatorBlob( + `/api/simulators/${encodeURIComponent(udid)}/screenshot.png${params}`, + ); +} + +export function recordSimulatorScreen( + udid: string, + seconds = 5, +): Promise { + return fetchSimulatorBlob( + `/api/simulators/${encodeURIComponent(udid)}/screen-recording`, + { + body: JSON.stringify({ seconds }), + method: "POST", + }, + ); +} diff --git a/client/src/app/AppShell.tsx b/client/src/app/AppShell.tsx index 5f4aff98..46032837 100644 --- a/client/src/app/AppShell.tsx +++ b/client/src/app/AppShell.tsx @@ -17,12 +17,14 @@ import { import { apiUrl, configureSimDeckClient } from "../api/config"; import { bootSimulator, + captureSimulatorScreenshot, dismissKeyboard, launchSimulatorBundle, openAppSwitcher, openSimulatorUrl, pressHome, pressSimulatorButton, + recordSimulatorScreen, rotateDigitalCrown, rotateRight, simulatorControlSocketUrl, @@ -268,6 +270,25 @@ function writeStreamTransportQueryParam(transport: StreamTransport) { ); } +function downloadBlob(blob: Blob, fileName: string) { + const url = URL.createObjectURL(blob); + const link = document.createElement("a"); + link.href = url; + link.download = fileName; + document.body.appendChild(link); + link.click(); + link.remove(); + URL.revokeObjectURL(url); +} + +function captureFileBaseName( + simulator: SimulatorMetadata, + artifact: "Recording" | "Screenshot", +): string { + const safeName = simulator.name.replace(/[^A-Za-z0-9._-]+/g, "-"); + return `SimDeck ${artifact} - ${safeName || simulator.udid}`; +} + function simulatorDisplaySize( simulator: SimulatorMetadata | null, ): Size | null { @@ -1746,6 +1767,48 @@ export function AppShell({ } } + async function downloadSimulatorScreenshot(withBezel: boolean) { + if (!selectedSimulator) { + return; + } + setLocalError(""); + try { + const blob = await captureSimulatorScreenshot(selectedSimulator.udid, { + withBezel, + }); + downloadBlob( + blob, + `${captureFileBaseName(selectedSimulator, "Screenshot")}${withBezel ? " Bezel" : ""}.png`, + ); + } catch (captureError) { + setLocalError( + captureError instanceof Error + ? captureError.message + : "Capture failed.", + ); + } + } + + async function downloadSimulatorRecording() { + if (!selectedSimulator) { + return; + } + setLocalError(""); + try { + const blob = await recordSimulatorScreen(selectedSimulator.udid, 5); + downloadBlob( + blob, + `${captureFileBaseName(selectedSimulator, "Recording")}.mp4`, + ); + } catch (captureError) { + setLocalError( + captureError instanceof Error + ? captureError.message + : "Recording failed.", + ); + } + } + function selectedStateFromSimulator( simulator: SimulatorMetadata, current: SimulatorStateResponse | null, @@ -2478,6 +2541,12 @@ export function AppShell({ bootSimulator(udid), ); }} + onCaptureScreenshot={() => { + void downloadSimulatorScreenshot(false); + }} + onCaptureScreenshotWithBezel={() => { + void downloadSimulatorScreenshot(true); + }} onChangeSearch={setSearch} onDismissKeyboard={() => { if (!selectedSimulator) { @@ -2546,6 +2615,9 @@ export function AppShell({ setRotationQuarterTurns((current) => (current + 1) % 4); }, false); }} + onRecordScreen={() => { + void downloadSimulatorRecording(); + }} onStreamEncoderChange={updateStreamEncoder} onStreamFpsChange={updateStreamFps} onStreamQualityChange={updateStreamQuality} diff --git a/client/src/features/simulators/SimulatorMenu.tsx b/client/src/features/simulators/SimulatorMenu.tsx index fe739271..f7f7aacd 100644 --- a/client/src/features/simulators/SimulatorMenu.tsx +++ b/client/src/features/simulators/SimulatorMenu.tsx @@ -23,6 +23,8 @@ interface SimulatorMenuProps { menuOpen: boolean; menuRef: RefObject; onBoot: () => void; + onCaptureScreenshot: () => void; + onCaptureScreenshotWithBezel: () => void; onChangeSearch: (value: string) => void; onCloseMenu: () => void; onDismissKeyboard: () => void; @@ -33,6 +35,7 @@ interface SimulatorMenuProps { onOpenNewSimulator: () => void; onOpenUrlPrompt: () => void; onRotateRight: () => void; + onRecordScreen: () => void; onShutdown: () => void; onStreamEncoderChange: (encoder: StreamEncoder) => void; onStreamFpsChange: (fps: StreamFps) => void; @@ -62,6 +65,8 @@ export function SimulatorMenu({ menuOpen, menuRef, onBoot, + onCaptureScreenshot, + onCaptureScreenshotWithBezel, onChangeSearch, onCloseMenu, onDismissKeyboard, @@ -72,6 +77,7 @@ export function SimulatorMenu({ onOpenNewSimulator, onOpenUrlPrompt, onRotateRight, + onRecordScreen, onShutdown, onStreamEncoderChange, onStreamFpsChange, @@ -280,6 +286,33 @@ export function SimulatorMenu({ + + +