diff --git a/README.md b/README.md index 416270ef..02d248af 100644 --- a/README.md +++ b/README.md @@ -41,6 +41,7 @@ view inside the editor. - Real-time screen `describe` command using accessibility view tree - available in token-efficient format for agents - Simulator app performance gauges for CPU, memory, disk writes, network throughput, hang signals, and stack sampling - CoreSimulator chrome asset rendering for device bezels +- iOS Simulator camera simulation from a generated pattern, local media file or stream URL, or a Mac camera source - NativeScript, React Native, Flutter, UIKit and SwiftUI runtime inspector plugins to debug app's view hierarchy live - `simdeck/test` for fast JS-based app tests that can query accessibility state and drive simulator controls @@ -126,6 +127,11 @@ simdeck install android: /path/to/app.apk simdeck uninstall com.example.App simdeck open-url https://example.com simdeck launch com.apple.Preferences +simdeck camera sources +simdeck camera start com.example.App --file /absolute/path/to/camera.mov +simdeck camera start com.example.App --webcam +simdeck camera switch --placeholder +simdeck camera stop simdeck toggle-appearance simdeck pasteboard set "hello" simdeck pasteboard get @@ -188,6 +194,14 @@ transport stream. `stream` writes an Annex B H.264 elementary stream to stdout for diagnostics or external tools such as `ffplay`. +`camera start` asks the SimDeck daemon to publish a camera feed, injects the +SimDeck camera shim into the target iOS simulator app, and relaunches that +bundle. The source can be a generated pattern, an absolute image or video path, +an `http://`, `https://`, or `file://` video URL, or a Mac camera selected with +`--webcam [id-or-name]`. +Use the browser menu item **Camera Simulation...** for the same flow from the UI. +Camera simulation is iOS-simulator-only and requires a booted simulator. + `describe` uses the project daemon to prefer React Native, NativeScript, Flutter, or UIKit in-app inspectors, then falls back to the built-in private CoreSimulator accessibility bridge. Use `--format agent` or diff --git a/cli/camera/SimDeckCameraInfo.plist b/cli/camera/SimDeckCameraInfo.plist new file mode 100644 index 00000000..ad60cc37 --- /dev/null +++ b/cli/camera/SimDeckCameraInfo.plist @@ -0,0 +1,12 @@ + + + + + CFBundleIdentifier + dev.nativescript.simdeck + CFBundleName + SimDeck + NSCameraUsageDescription + SimDeck can forward the Mac camera into iOS Simulator apps for local camera testing. + + diff --git a/cli/camera/SimDeckCameraInjector.m b/cli/camera/SimDeckCameraInjector.m new file mode 100644 index 00000000..99721c48 --- /dev/null +++ b/cli/camera/SimDeckCameraInjector.m @@ -0,0 +1,1066 @@ +#import +#import +#import +#import +#import +#import +#import + +#import "SimDeckCameraShared.h" + +#import +#import +#import +#import +#import +#import +#import +#import +#import +#import +#import + +static SimDeckCameraHeader *gHeader; +static uint8_t *gFrameBytes; +static size_t gFrameMapSize; +static dispatch_source_t gFrameTimer; +static dispatch_queue_t gFrameQueue; +static NSMutableArray *gSessions; +static NSMutableArray *gVideoOutputs; +static NSHashTable *gPreviewLayers; +static NSMutableSet *gHookedVideoOutputClasses; + +static char kSessionInputsKey; +static char kSessionOutputsKey; +static char kSessionRunningKey; +static char kInputFakeKey; +static char kInputDeviceKey; +static char kOutputDelegateKey; +static char kOutputQueueKey; +static char kOutputVideoSettingsKey; +static char kOutputDiscardsLateFramesKey; +static char kPreviewOverlayKey; +static char kPreviewHostKey; +static char kPickerOverlayViewKey; +static char kPickerCaptureControlKey; +static char kPickerCameraOverlayKey; +static char kPickerCaptureWindowKey; + +static void StartFrameTimer(void); +static void InstallVideoOutputDelegateHook(Class cls); +static void Log(NSString *format, ...); +static void DebugLog(NSString *format, ...); +static BOOL OpenSharedCamera(void); +static void TrackPointer(NSMutableArray *pointers, id object); +static void RegisterOutputLayer(CALayer *layer); +static void RegisterPreviewLayer(CALayer *layer); +static void SendPickerCapture(UIImagePickerController *picker); + +static BOOL IsVideoMediaType(AVMediaType mediaType) { + return mediaType == nil || [mediaType isEqualToString:AVMediaTypeVideo]; +} + +static void SimDeckSetSampleBufferDelegate(AVCaptureVideoDataOutput *output, + SEL selector, + id delegate, + dispatch_queue_t sampleBufferCallbackQueue) { + (void)selector; + objc_setAssociatedObject(output, &kOutputDelegateKey, delegate, OBJC_ASSOCIATION_ASSIGN); + objc_setAssociatedObject(output, &kOutputQueueKey, sampleBufferCallbackQueue, OBJC_ASSOCIATION_RETAIN_NONATOMIC); + @synchronized(gVideoOutputs) { + TrackPointer(gVideoOutputs, output); + } + StartFrameTimer(); +} + +static void TrackPointer(NSMutableArray *pointers, id object) { + if (!pointers || !object) return; + void *pointer = (__bridge void *)object; + for (NSValue *value in pointers) { + if (value.pointerValue == pointer) return; + } + [pointers addObject:[NSValue valueWithPointer:pointer]]; +} + +static void RegisterOutputLayer(CALayer *layer) { + if (!layer) return; + @synchronized(gPreviewLayers) { + [gPreviewLayers addObject:layer]; + } + layer.contentsGravity = kCAGravityResizeAspectFill; + layer.masksToBounds = YES; + StartFrameTimer(); +} + +static void RegisterPreviewLayer(CALayer *layer) { + if (!layer) return; + CALayer *overlay = objc_getAssociatedObject(layer, &kPreviewOverlayKey); + if (!overlay) { + overlay = [CALayer layer]; + overlay.contentsGravity = kCAGravityResizeAspectFill; + overlay.masksToBounds = YES; + overlay.frame = layer.bounds; + objc_setAssociatedObject(overlay, &kPreviewHostKey, layer, OBJC_ASSOCIATION_ASSIGN); + objc_setAssociatedObject(layer, &kPreviewOverlayKey, overlay, OBJC_ASSOCIATION_RETAIN_NONATOMIC); + [layer addSublayer:overlay]; + DebugLog(@"installed preview frame layer on %@", NSStringFromClass(object_getClass(layer))); + } + RegisterOutputLayer(overlay); +} + +static void SimDeckSetVideoSettings(AVCaptureVideoDataOutput *output, + SEL selector, + NSDictionary *videoSettings) { + (void)selector; + objc_setAssociatedObject(output, &kOutputVideoSettingsKey, videoSettings, OBJC_ASSOCIATION_RETAIN_NONATOMIC); + InstallVideoOutputDelegateHook(object_getClass(output)); + DebugLog(@"captured video settings on %@", NSStringFromClass(object_getClass(output))); +} + +static id SimDeckVideoDataOutputAllocWithZone(Class cls, SEL selector, struct _NSZone *zone) { + (void)selector; + (void)zone; + if (cls == AVCaptureVideoDataOutput.class && OpenSharedCamera()) { + Class fakeClass = objc_getClass("SimDeckCameraVideoDataOutput"); + if (fakeClass) return class_createInstance(fakeClass, 0); + } + struct objc_super superInfo = { + .receiver = cls, + .super_class = class_getSuperclass(object_getClass(cls)), + }; + return ((id (*)(struct objc_super *, SEL, struct _NSZone *))objc_msgSendSuper)(&superInfo, @selector(allocWithZone:), zone); +} + +static void Log(NSString *format, ...) { + va_list args; + va_start(args, format); + NSString *message = [[NSString alloc] initWithFormat:format arguments:args]; + va_end(args); + fprintf(stderr, "[simdeck-camera] %s\n", message.UTF8String ?: ""); +} + +static BOOL DebugLoggingEnabled(void) { + static BOOL enabled; + static dispatch_once_t once; + dispatch_once(&once, ^{ + const char *value = getenv("SIMDECK_CAMERA_DEBUG"); + enabled = value && value[0] != '\0' && strcmp(value, "0") != 0; + }); + return enabled; +} + +static void DebugLog(NSString *format, ...) { + if (!DebugLoggingEnabled()) return; + va_list args; + va_start(args, format); + NSString *message = [[NSString alloc] initWithFormat:format arguments:args]; + va_end(args); + fprintf(stderr, "[simdeck-camera] %s\n", message.UTF8String ?: ""); +} + +static BOOL OpenSharedCamera(void) { + if (gHeader) return YES; + const char *name = getenv("SIMDECK_CAMERA_SHM_NAME"); + if (!name || name[0] == '\0') { + name = getenv("SIMCAM_SHM_NAME"); + } + if (!name || name[0] == '\0') { + return NO; + } + int fd = shm_open(name, O_RDONLY, 0); + if (fd < 0) { + Log(@"unable to open shared memory %s", name); + return NO; + } + struct stat st; + if (fstat(fd, &st) != 0 || st.st_size < (off_t)SIMDECK_CAMERA_HEADER_SIZE) { + close(fd); + return NO; + } + void *mapped = mmap(NULL, (size_t)st.st_size, PROT_READ, MAP_SHARED, fd, 0); + close(fd); + if (mapped == MAP_FAILED) { + return NO; + } + SimDeckCameraHeader *header = (SimDeckCameraHeader *)mapped; + if (header->magic != SIMDECK_CAMERA_MAGIC || header->version != SIMDECK_CAMERA_VERSION) { + munmap(mapped, (size_t)st.st_size); + return NO; + } + gHeader = header; + gFrameMapSize = (size_t)st.st_size; + gFrameBytes = ((uint8_t *)mapped) + header->headerSize; + DebugLog(@"attached shared feed %ux%u", header->width, header->height); + return YES; +} + +static BOOL CopyCurrentFrame(NSMutableData **outData, uint32_t *outWidth, uint32_t *outHeight, uint32_t *outBytesPerRow, uint64_t *outTimestampNs) { + if (!OpenSharedCamera()) return NO; + uint32_t width = gHeader->width; + uint32_t height = gHeader->height; + uint32_t bytesPerRow = gHeader->bytesPerRow; + if (width == 0 || height == 0 || bytesPerRow < width * 4) return NO; + size_t length = (size_t)bytesPerRow * height; + if ((size_t)gHeader->headerSize + length > gFrameMapSize) return NO; + + NSMutableData *copy = [NSMutableData dataWithLength:length]; + uint64_t before = 0; + uint64_t after = 0; + for (int attempt = 0; attempt < 4; attempt += 1) { + before = gHeader->sequence; + if ((before & 1u) != 0) { + usleep(1000); + continue; + } + memcpy(copy.mutableBytes, gFrameBytes, length); + after = gHeader->sequence; + if (before == after && (after & 1u) == 0) { + *outData = copy; + *outWidth = width; + *outHeight = height; + *outBytesPerRow = bytesPerRow; + *outTimestampNs = gHeader->timestampNs; + return YES; + } + } + return NO; +} + +static CMSampleBufferRef CreateSampleBuffer(NSData *frameData, uint32_t width, uint32_t height, uint32_t bytesPerRow, uint64_t timestampNs) { + CVPixelBufferRef pixelBuffer = NULL; + NSDictionary *attrs = @{ + (id)kCVPixelBufferIOSurfacePropertiesKey: @{}, + (id)kCVPixelBufferCGImageCompatibilityKey: @YES, + (id)kCVPixelBufferCGBitmapContextCompatibilityKey: @YES, + }; + CVReturn cv = CVPixelBufferCreate(kCFAllocatorDefault, + width, + height, + kCVPixelFormatType_32BGRA, + (__bridge CFDictionaryRef)attrs, + &pixelBuffer); + if (cv != kCVReturnSuccess || !pixelBuffer) return NULL; + CVPixelBufferLockBaseAddress(pixelBuffer, 0); + uint8_t *dest = CVPixelBufferGetBaseAddress(pixelBuffer); + size_t destBytesPerRow = CVPixelBufferGetBytesPerRow(pixelBuffer); + const uint8_t *source = frameData.bytes; + for (uint32_t y = 0; y < height; y += 1) { + memcpy(dest + (size_t)y * destBytesPerRow, + source + (size_t)y * bytesPerRow, + MIN(destBytesPerRow, bytesPerRow)); + } + CVPixelBufferUnlockBaseAddress(pixelBuffer, 0); + + CMVideoFormatDescriptionRef format = NULL; + OSStatus status = CMVideoFormatDescriptionCreateForImageBuffer(kCFAllocatorDefault, pixelBuffer, &format); + if (status != noErr || !format) { + CVPixelBufferRelease(pixelBuffer); + return NULL; + } + CMTime pts = CMTimeMake((int64_t)(timestampNs ?: (uint64_t)(CACurrentMediaTime() * 1000000000.0)), 1000000000); + CMSampleTimingInfo timing = { + .duration = CMTimeMake(1, 30), + .presentationTimeStamp = pts, + .decodeTimeStamp = kCMTimeInvalid, + }; + CMSampleBufferRef sample = NULL; + status = CMSampleBufferCreateReadyWithImageBuffer(kCFAllocatorDefault, + pixelBuffer, + format, + &timing, + &sample); + CFRelease(format); + CVPixelBufferRelease(pixelBuffer); + return status == noErr ? sample : NULL; +} + +static CGImageRef CreateImage(NSData *frameData, uint32_t width, uint32_t height, uint32_t bytesPerRow) { + CGColorSpaceRef colorSpace = CGColorSpaceCreateDeviceRGB(); + CGDataProviderRef provider = CGDataProviderCreateWithCFData((__bridge CFDataRef)frameData); + CGImageRef image = CGImageCreate(width, + height, + 8, + 32, + bytesPerRow, + colorSpace, + kCGBitmapByteOrder32Little | kCGImageAlphaPremultipliedFirst, + provider, + NULL, + false, + kCGRenderingIntentDefault); + CGDataProviderRelease(provider); + CGColorSpaceRelease(colorSpace); + return image; +} + +static UIImage *CurrentFrameImage(void) { + NSMutableData *data = nil; + uint32_t width = 0; + uint32_t height = 0; + uint32_t bytesPerRow = 0; + uint64_t timestampNs = 0; + if (!CopyCurrentFrame(&data, &width, &height, &bytesPerRow, ×tampNs)) return nil; + CGImageRef image = CreateImage(data, width, height, bytesPerRow); + if (!image) return nil; + UIImage *uiImage = [UIImage imageWithCGImage:image scale:UIScreen.mainScreen.scale orientation:UIImageOrientationUp]; + CGImageRelease(image); + return uiImage; +} + +static void DeliverFrame(void) { + NSMutableData *data = nil; + uint32_t width = 0; + uint32_t height = 0; + uint32_t bytesPerRow = 0; + uint64_t timestampNs = 0; + if (!CopyCurrentFrame(&data, &width, &height, &bytesPerRow, ×tampNs)) return; + timestampNs = (uint64_t)(CACurrentMediaTime() * 1000000000.0); + + CMSampleBufferRef sample = CreateSampleBuffer(data, width, height, bytesPerRow, timestampNs); + if (sample) { + NSArray *outputs = nil; + @synchronized(gVideoOutputs) { + outputs = [gVideoOutputs copy]; + } + for (NSValue *value in outputs) { + AVCaptureVideoDataOutput *output = (__bridge AVCaptureVideoDataOutput *)value.pointerValue; + id delegate = objc_getAssociatedObject(output, &kOutputDelegateKey); + dispatch_queue_t queue = objc_getAssociatedObject(output, &kOutputQueueKey); + if (!delegate || ![delegate respondsToSelector:@selector(captureOutput:didOutputSampleBuffer:fromConnection:)]) { + continue; + } + CFRetain(sample); + dispatch_async(queue ?: dispatch_get_main_queue(), ^{ + ((void (*)(id, SEL, AVCaptureOutput *, CMSampleBufferRef, AVCaptureConnection *))objc_msgSend)( + delegate, + @selector(captureOutput:didOutputSampleBuffer:fromConnection:), + output, + sample, + nil); + CFRelease(sample); + }); + } + [outputs release]; + CFRelease(sample); + } + + CGImageRef image = CreateImage(data, width, height, bytesPerRow); + if (image) { + NSArray *layers = nil; + @synchronized(gPreviewLayers) { + layers = gPreviewLayers.allObjects; + } + dispatch_async(dispatch_get_main_queue(), ^{ + for (CALayer *layer in layers) { + CALayer *host = objc_getAssociatedObject(layer, &kPreviewHostKey); + if (host) { + layer.frame = host.bounds; + } + layer.contents = (__bridge id)image; + layer.contentsGravity = kCAGravityResizeAspectFill; + if (gHeader && gHeader->mirrorMode == SIMDECK_CAMERA_MIRROR_ON) { + layer.transform = CATransform3DMakeScale(-1, 1, 1); + } else { + layer.transform = CATransform3DIdentity; + } + } + CGImageRelease(image); + }); + } +} + +static void StartFrameTimer(void) { + if (gFrameTimer) return; + if (!gFrameQueue) { + gFrameQueue = dispatch_queue_create("dev.nativescript.simdeck.camera.injector", DISPATCH_QUEUE_SERIAL); + } + gFrameTimer = dispatch_source_create(DISPATCH_SOURCE_TYPE_TIMER, 0, 0, gFrameQueue); + dispatch_source_set_timer(gFrameTimer, + dispatch_time(DISPATCH_TIME_NOW, 0), + (uint64_t)(NSEC_PER_SEC / 30), + (uint64_t)(NSEC_PER_MSEC * 4)); + dispatch_source_set_event_handler(gFrameTimer, ^{ + DeliverFrame(); + }); + dispatch_resume(gFrameTimer); +} + +static void AddSessionOutput(AVCaptureSession *session, AVCaptureOutput *output) { + NSMutableArray *outputs = objc_getAssociatedObject(session, &kSessionOutputsKey); + if (!outputs) { + outputs = [NSMutableArray array]; + objc_setAssociatedObject(session, &kSessionOutputsKey, outputs, OBJC_ASSOCIATION_RETAIN_NONATOMIC); + } + if (![outputs containsObject:output]) [outputs addObject:output]; + if ([output isKindOfClass:AVCaptureVideoDataOutput.class]) { + @synchronized(gVideoOutputs) { + TrackPointer(gVideoOutputs, output); + } + } +} + +static BOOL SimDeckIsFakeInput(id input) { + return [objc_getAssociatedObject(input, &kInputFakeKey) boolValue]; +} + +static AVCaptureDeviceInput *SimDeckFakeInput(void) { + static AVCaptureDeviceInput *input; + static dispatch_once_t once; + dispatch_once(&once, ^{ + input = (AVCaptureDeviceInput *)class_createInstance(AVCaptureDeviceInput.class, 0); + objc_setAssociatedObject(input, &kInputFakeKey, @YES, OBJC_ASSOCIATION_RETAIN_NONATOMIC); + Class deviceClass = objc_getClass("SimDeckCameraDevice"); + id device = ((id (*)(Class, SEL))objc_msgSend)(deviceClass, @selector(sharedDevice)); + objc_setAssociatedObject(input, &kInputDeviceKey, device, OBJC_ASSOCIATION_RETAIN_NONATOMIC); + }); + return input; +} + +static BOOL ClassIsSubclassOf(Class cls, Class parent) { + for (Class current = cls; current; current = class_getSuperclass(current)) { + if (current == parent) return YES; + } + return NO; +} + +static void InstallVideoOutputDelegateHook(Class cls) { + if (!cls) return; + @synchronized(gHookedVideoOutputClasses) { + NSString *name = NSStringFromClass(cls); + if ([gHookedVideoOutputClasses containsObject:name]) return; + [gHookedVideoOutputClasses addObject:name]; + } + Method original = class_getInstanceMethod(cls, @selector(setSampleBufferDelegate:queue:)); + if (!original) { + Log(@"missing video output delegate hook on %@", NSStringFromClass(cls)); + return; + } + class_replaceMethod(cls, + @selector(setSampleBufferDelegate:queue:), + (IMP)SimDeckSetSampleBufferDelegate, + method_getTypeEncoding(original)); + Method settings = class_getInstanceMethod(cls, @selector(setVideoSettings:)); + if (settings) { + class_replaceMethod(cls, + @selector(setVideoSettings:), + (IMP)SimDeckSetVideoSettings, + method_getTypeEncoding(settings)); + } + DebugLog(@"hooked video output delegate on %@", NSStringFromClass(cls)); +} + +static void InstallExistingVideoOutputDelegateHooks(void) { + int count = objc_getClassList(NULL, 0); + if (count <= 0) return; + Class *classes = calloc((size_t)count, sizeof(Class)); + if (!classes) return; + count = objc_getClassList(classes, count); + for (int index = 0; index < count; index += 1) { + Class cls = classes[index]; + if (ClassIsSubclassOf(cls, AVCaptureVideoDataOutput.class)) { + InstallVideoOutputDelegateHook(cls); + } + } + free(classes); +} + +static void InstallVideoOutputAllocationHook(void) { + Method alloc = class_getClassMethod(AVCaptureVideoDataOutput.class, @selector(allocWithZone:)); + Class meta = object_getClass(AVCaptureVideoDataOutput.class); + if (!alloc || !meta) { + Log(@"missing video output allocation hook"); + return; + } + class_replaceMethod(meta, + @selector(allocWithZone:), + (IMP)SimDeckVideoDataOutputAllocWithZone, + method_getTypeEncoding(alloc)); + DebugLog(@"hooked video output allocation"); +} + +@interface SimDeckCameraDevice : AVCaptureDevice +@end + +@implementation SimDeckCameraDevice + ++ (instancetype)sharedDevice { + static SimDeckCameraDevice *device; + static dispatch_once_t once; + dispatch_once(&once, ^{ + device = NSAllocateObject(self, 0, nil); + }); + return device; +} + +- (NSString *)localizedName { return @"SimDeck Camera"; } +- (NSString *)uniqueID { return @"dev.nativescript.simdeck.camera"; } +- (AVCaptureDevicePosition)position { return AVCaptureDevicePositionBack; } +- (BOOL)hasMediaType:(AVMediaType)mediaType { return IsVideoMediaType(mediaType); } +- (BOOL)isConnected { return YES; } +- (BOOL)isSuspended { return NO; } +- (AVCaptureDeviceType)deviceType { return AVCaptureDeviceTypeBuiltInWideAngleCamera; } + +@end + +@interface SimDeckCameraInput : AVCaptureDeviceInput +@end + +@implementation SimDeckCameraInput + +- (AVCaptureDevice *)device { + return (AVCaptureDevice *)[SimDeckCameraDevice sharedDevice]; +} + +@end + +@interface SimDeckCameraVideoDataOutput : AVCaptureVideoDataOutput +@end + +@implementation SimDeckCameraVideoDataOutput + +#pragma clang diagnostic push +#pragma clang diagnostic ignored "-Wobjc-designated-initializers" +- (instancetype)init { + struct objc_super superInfo = { self, NSObject.class }; + return ((id (*)(struct objc_super *, SEL))objc_msgSendSuper)(&superInfo, @selector(init)); +} +#pragma clang diagnostic pop + +- (void)setVideoSettings:(NSDictionary *)videoSettings { + objc_setAssociatedObject(self, &kOutputVideoSettingsKey, videoSettings, OBJC_ASSOCIATION_RETAIN_NONATOMIC); +} + +- (NSDictionary *)videoSettings { + return objc_getAssociatedObject(self, &kOutputVideoSettingsKey); +} + +- (void)setAlwaysDiscardsLateVideoFrames:(BOOL)alwaysDiscardsLateVideoFrames { + objc_setAssociatedObject(self, &kOutputDiscardsLateFramesKey, @(alwaysDiscardsLateVideoFrames), OBJC_ASSOCIATION_RETAIN_NONATOMIC); +} + +- (BOOL)alwaysDiscardsLateVideoFrames { + NSNumber *value = objc_getAssociatedObject(self, &kOutputDiscardsLateFramesKey); + return value ? value.boolValue : YES; +} + +- (void)setSampleBufferDelegate:(id)delegate queue:(dispatch_queue_t)sampleBufferCallbackQueue { + DebugLog(@"fake video output delegate set"); + SimDeckSetSampleBufferDelegate(self, _cmd, delegate, sampleBufferCallbackQueue); +} + +@end + +@interface SimDeckCameraPhoto : AVCapturePhoto +@property (nonatomic, strong) NSData *jpegData; +@end + +@implementation SimDeckCameraPhoto + +- (NSData *)fileDataRepresentation { + return self.jpegData; +} + +@end + +@implementation AVCaptureDevice (SimDeckCamera) + ++ (AVCaptureDevice *)sd_defaultDeviceWithMediaType:(AVMediaType)mediaType { + AVCaptureDevice *device = [self sd_defaultDeviceWithMediaType:mediaType]; + if (device || !IsVideoMediaType(mediaType) || !OpenSharedCamera()) return device; + return (AVCaptureDevice *)[SimDeckCameraDevice sharedDevice]; +} + ++ (AVCaptureDevice *)sd_defaultDeviceWithDeviceType:(AVCaptureDeviceType)deviceType + mediaType:(AVMediaType)mediaType + position:(AVCaptureDevicePosition)position { + AVCaptureDevice *device = [self sd_defaultDeviceWithDeviceType:deviceType mediaType:mediaType position:position]; + if (device || !IsVideoMediaType(mediaType) || !OpenSharedCamera()) return device; + return (AVCaptureDevice *)[SimDeckCameraDevice sharedDevice]; +} + ++ (NSArray *)sd_devicesWithMediaType:(AVMediaType)mediaType { + NSArray *devices = [self sd_devicesWithMediaType:mediaType]; + if (devices.count > 0 || !IsVideoMediaType(mediaType)) return devices; + return OpenSharedCamera() ? @[ (AVCaptureDevice *)[SimDeckCameraDevice sharedDevice] ] : @[]; +} + ++ (AVAuthorizationStatus)sd_authorizationStatusForMediaType:(AVMediaType)mediaType { + if (IsVideoMediaType(mediaType)) return AVAuthorizationStatusAuthorized; + return [self sd_authorizationStatusForMediaType:mediaType]; +} + ++ (void)sd_requestAccessForMediaType:(AVMediaType)mediaType completionHandler:(void (^)(BOOL granted))handler { + if (IsVideoMediaType(mediaType)) { + if (handler) dispatch_async(dispatch_get_main_queue(), ^{ handler(YES); }); + return; + } + [self sd_requestAccessForMediaType:mediaType completionHandler:handler]; +} + +@end + +@implementation AVCaptureDeviceDiscoverySession (SimDeckCamera) + +- (NSArray *)sd_devices { + NSArray *devices = [self sd_devices]; + if (devices.count > 0) return devices; + return OpenSharedCamera() ? @[ (AVCaptureDevice *)[SimDeckCameraDevice sharedDevice] ] : @[]; +} + +@end + +@implementation AVCaptureDeviceInput (SimDeckCamera) + ++ (instancetype)sd_deviceInputWithDevice:(AVCaptureDevice *)device error:(NSError **)outError { + if ([device isKindOfClass:SimDeckCameraDevice.class]) { + if (outError) *outError = nil; + return (id)SimDeckFakeInput(); + } + id input = [self sd_deviceInputWithDevice:device error:outError]; + if (!input && OpenSharedCamera() && [device hasMediaType:AVMediaTypeVideo]) { + if (outError) *outError = nil; + return (id)SimDeckFakeInput(); + } + return input; +} + +- (instancetype)sd_initWithDevice:(AVCaptureDevice *)device error:(NSError **)outError { + if ([device isKindOfClass:SimDeckCameraDevice.class]) { + if (outError) *outError = nil; + return (id)SimDeckFakeInput(); + } + return [self sd_initWithDevice:device error:outError]; +} + +- (AVCaptureDevice *)sd_device { + AVCaptureDevice *device = objc_getAssociatedObject(self, &kInputDeviceKey); + return device ?: [self sd_device]; +} + +- (NSArray *)sd_ports { + if (SimDeckIsFakeInput(self)) return @[]; + return [self sd_ports]; +} + +@end + +@implementation AVCaptureVideoDataOutput (SimDeckCamera) + ++ (id)sd_allocWithZone:(struct _NSZone *)zone { + if (self == AVCaptureVideoDataOutput.class && OpenSharedCamera()) { + return NSAllocateObject(SimDeckCameraVideoDataOutput.class, 0, nil); + } + return [self sd_allocWithZone:zone]; +} + +- (instancetype)sd_init { + id output = [self sd_init]; + InstallVideoOutputDelegateHook(object_getClass(output)); + return output; +} + +- (void)sd_setSampleBufferDelegate:(id)delegate queue:(dispatch_queue_t)sampleBufferCallbackQueue { + SimDeckSetSampleBufferDelegate(self, _cmd, delegate, sampleBufferCallbackQueue); +} + +@end + +@implementation AVCapturePhotoOutput (SimDeckCamera) + +- (void)sd_capturePhotoWithSettings:(AVCapturePhotoSettings *)settings delegate:(id)delegate { + (void)settings; + NSMutableData *data = nil; + uint32_t width = 0; + uint32_t height = 0; + uint32_t bytesPerRow = 0; + uint64_t ts = 0; + if (!delegate || !CopyCurrentFrame(&data, &width, &height, &bytesPerRow, &ts)) { + [self sd_capturePhotoWithSettings:settings delegate:delegate]; + return; + } + CGImageRef image = CreateImage(data, width, height, bytesPerRow); + NSData *jpeg = image ? UIImageJPEGRepresentation([UIImage imageWithCGImage:image], 0.92) : nil; + if (image) CGImageRelease(image); + SimDeckCameraPhoto *photo = NSAllocateObject(SimDeckCameraPhoto.class, 0, nil); + photo.jpegData = jpeg ?: [NSData data]; + dispatch_async(dispatch_get_main_queue(), ^{ + if ([delegate respondsToSelector:@selector(captureOutput:didFinishProcessingPhoto:error:)]) { + [delegate captureOutput:self didFinishProcessingPhoto:photo error:nil]; + } + }); +} + +@end + +@implementation AVCaptureSession (SimDeckCamera) + +- (BOOL)sd_canAddInput:(AVCaptureInput *)input { + if (SimDeckIsFakeInput(input)) return YES; + return [self sd_canAddInput:input]; +} + +- (void)sd_addInput:(AVCaptureInput *)input { + if (SimDeckIsFakeInput(input)) { + NSMutableArray *inputs = objc_getAssociatedObject(self, &kSessionInputsKey); + if (!inputs) { + inputs = [NSMutableArray array]; + objc_setAssociatedObject(self, &kSessionInputsKey, inputs, OBJC_ASSOCIATION_RETAIN_NONATOMIC); + } + if (![inputs containsObject:input]) [inputs addObject:input]; + return; + } + [self sd_addInput:input]; +} + +- (BOOL)sd_canAddOutput:(AVCaptureOutput *)output { + if ([output isKindOfClass:AVCaptureVideoDataOutput.class] || [output isKindOfClass:AVCapturePhotoOutput.class]) return YES; + return [self sd_canAddOutput:output]; +} + +- (void)sd_addOutput:(AVCaptureOutput *)output { + if ([output isKindOfClass:AVCaptureVideoDataOutput.class] || [output isKindOfClass:AVCapturePhotoOutput.class]) { + AddSessionOutput(self, output); + return; + } + [self sd_addOutput:output]; +} + +- (NSArray *)sd_inputs { + NSArray *original = [self sd_inputs]; + NSMutableArray *inputs = objc_getAssociatedObject(self, &kSessionInputsKey); + if (inputs.count == 0) return original; + return [original arrayByAddingObjectsFromArray:inputs]; +} + +- (NSArray *)sd_outputs { + NSArray *original = [self sd_outputs]; + NSMutableArray *outputs = objc_getAssociatedObject(self, &kSessionOutputsKey); + if (outputs.count == 0) return original; + return [original arrayByAddingObjectsFromArray:outputs]; +} + +- (void)sd_startRunning { + objc_setAssociatedObject(self, &kSessionRunningKey, @YES, OBJC_ASSOCIATION_RETAIN_NONATOMIC); + @synchronized(gSessions) { + TrackPointer(gSessions, self); + } + StartFrameTimer(); +} + +- (void)sd_stopRunning { + objc_setAssociatedObject(self, &kSessionRunningKey, @NO, OBJC_ASSOCIATION_RETAIN_NONATOMIC); +} + +- (BOOL)sd_isRunning { + NSNumber *running = objc_getAssociatedObject(self, &kSessionRunningKey); + if (running) return running.boolValue; + return [self sd_isRunning]; +} + +@end + +@interface SimDeckPickerOverlayView : UIView +@end + +@implementation SimDeckPickerOverlayView + +- (UIView *)hitTest:(CGPoint)point withEvent:(UIEvent *)event { + UIView *hit = [super hitTest:point withEvent:event]; + return hit == self ? nil : hit; +} + +@end + +@interface SimDeckPickerOverlayWindow : UIWindow +@end + +@implementation SimDeckPickerOverlayWindow + +- (UIView *)hitTest:(CGPoint)point withEvent:(UIEvent *)event { + UIView *hit = [super hitTest:point withEvent:event]; + return hit == self || hit == self.rootViewController.view ? nil : hit; +} + +@end + +@interface SimDeckPickerCaptureControl : UIControl +@property (nonatomic, assign) UIImagePickerController *picker; +- (instancetype)initWithPicker:(UIImagePickerController *)picker; +- (void)capture; +@end + +@implementation SimDeckPickerCaptureControl + +- (instancetype)initWithPicker:(UIImagePickerController *)picker { + self = [super initWithFrame:CGRectZero]; + if (self) { + self.picker = picker; + self.backgroundColor = UIColor.clearColor; + self.accessibilityLabel = @"SimDeck Capture"; + [self addTarget:self action:@selector(capture) forControlEvents:UIControlEventTouchUpInside]; + } + return self; +} + +- (void)capture { + UIImagePickerController *picker = [self.picker retain]; + dispatch_async(dispatch_get_main_queue(), ^{ + SendPickerCapture(picker); + [picker release]; + }); +} + +@end + +static CGRect PickerCaptureFrame(UIView *view) { + CGRect bounds = view.bounds; + CGFloat width = CGRectGetWidth(bounds); + CGFloat height = CGRectGetHeight(bounds); + CGFloat size = MAX((CGFloat)104.0, MIN(width, height) * (CGFloat)0.26); + if (width > height) { + CGFloat centerX = width - MAX((CGFloat)120.0, height * (CGFloat)0.24); + CGFloat centerY = height * (CGFloat)0.5; + return CGRectMake(centerX - size * (CGFloat)0.5, centerY - size * (CGFloat)0.5, size, size); + } + CGFloat centerX = width * (CGFloat)0.5; + CGFloat centerY = height - MAX((CGFloat)140.0, width * (CGFloat)0.35); + return CGRectMake(centerX - size * (CGFloat)0.5, centerY - size * (CGFloat)0.5, size, size); +} + +static void SendPickerCapture(UIImagePickerController *picker) { + if (!picker) return; + UIImage *image = CurrentFrameImage(); + if (!image) return; + DebugLog(@"sending UIImagePicker simulated capture"); + NSDictionary *info = @{ + UIImagePickerControllerMediaType: @"public.image", + UIImagePickerControllerOriginalImage: image, + UIImagePickerControllerMediaMetadata: @{}, + }; + id delegate = picker.delegate; + if ([delegate respondsToSelector:@selector(imagePickerController:didFinishPickingMediaWithInfo:)]) { + [delegate imagePickerController:picker didFinishPickingMediaWithInfo:info]; + } else { + [picker dismissViewControllerAnimated:YES completion:nil]; + } +} + +static CGRect PickerPreviewFrame(UIView *view) { + CGRect bounds = view.bounds; + CGFloat width = CGRectGetWidth(bounds); + CGFloat height = CGRectGetHeight(bounds); + if (width <= 0 || height <= 0) return bounds; + if (width > height) { + CGFloat previewWidth = MIN(width, height * 4.0 / 3.0); + CGFloat x = (width - previewWidth) * 0.5; + return CGRectMake(x, 0, previewWidth, height); + } + CGFloat previewHeight = MIN(height, width * 4.0 / 3.0); + CGFloat bottomControls = MAX((CGFloat)150.0, width * (CGFloat)0.48); + CGFloat y = height - previewHeight - bottomControls; + if (y < 0) { + y = (height - previewHeight) * 0.5; + } + y = MAX((CGFloat)0.0, MIN(y, height - previewHeight)); + return CGRectMake(0, y, width, previewHeight); +} + +static void InstallPickerOverlay(UIImagePickerController *picker) { + if (!picker || !OpenSharedCamera()) return; + if (picker.sourceType != UIImagePickerControllerSourceTypeCamera) return; + UIView *root = picker.view; + if (!root) return; + + UIView *overlay = objc_getAssociatedObject(picker, &kPickerOverlayViewKey); + if (!overlay) { + overlay = [[[UIView alloc] initWithFrame:PickerPreviewFrame(root)] autorelease]; + overlay.userInteractionEnabled = NO; + overlay.clipsToBounds = YES; + overlay.backgroundColor = UIColor.clearColor; + overlay.layer.contentsGravity = kCAGravityResizeAspectFill; + overlay.layer.masksToBounds = YES; + objc_setAssociatedObject(picker, &kPickerOverlayViewKey, overlay, OBJC_ASSOCIATION_RETAIN_NONATOMIC); + [root addSubview:overlay]; + DebugLog(@"installed UIImagePicker camera preview overlay"); + } + overlay.frame = PickerPreviewFrame(root); + RegisterOutputLayer(overlay.layer); + + SimDeckPickerOverlayView *cameraOverlay = objc_getAssociatedObject(picker, &kPickerCameraOverlayKey); + if (!cameraOverlay) { + cameraOverlay = [[[SimDeckPickerOverlayView alloc] initWithFrame:root.bounds] autorelease]; + cameraOverlay.backgroundColor = UIColor.clearColor; + cameraOverlay.autoresizingMask = UIViewAutoresizingFlexibleWidth | UIViewAutoresizingFlexibleHeight; + objc_setAssociatedObject(picker, &kPickerCameraOverlayKey, cameraOverlay, OBJC_ASSOCIATION_RETAIN_NONATOMIC); + picker.cameraOverlayView = cameraOverlay; + } + cameraOverlay.frame = root.bounds; + + UIWindow *hostWindow = root.window; + if (!hostWindow) { + for (UIScene *scene in UIApplication.sharedApplication.connectedScenes) { + if (![scene isKindOfClass:UIWindowScene.class]) continue; + for (UIWindow *window in ((UIWindowScene *)scene).windows) { + if (window.isKeyWindow) { + hostWindow = window; + break; + } + } + if (hostWindow) break; + } + } + CGRect captureFrame = PickerCaptureFrame(root); + SimDeckPickerOverlayWindow *captureWindow = objc_getAssociatedObject(picker, &kPickerCaptureWindowKey); + if (!captureWindow) { + captureWindow = [[[SimDeckPickerOverlayWindow alloc] initWithFrame:captureFrame] autorelease]; + captureWindow.windowLevel = UIWindowLevelAlert + 10.0; + captureWindow.backgroundColor = UIColor.clearColor; + UIViewController *rootController = [[[UIViewController alloc] init] autorelease]; + rootController.view = [[[SimDeckPickerOverlayView alloc] initWithFrame:captureWindow.bounds] autorelease]; + rootController.view.backgroundColor = UIColor.clearColor; + rootController.view.autoresizingMask = UIViewAutoresizingFlexibleWidth | UIViewAutoresizingFlexibleHeight; + captureWindow.rootViewController = rootController; + captureWindow.hidden = NO; + objc_setAssociatedObject(picker, &kPickerCaptureWindowKey, captureWindow, OBJC_ASSOCIATION_RETAIN_NONATOMIC); + } + if (@available(iOS 13.0, *)) { + if (hostWindow.windowScene && captureWindow.windowScene != hostWindow.windowScene) { + captureWindow.windowScene = hostWindow.windowScene; + } + } + captureWindow.frame = captureFrame; + captureWindow.hidden = NO; + UIView *captureRoot = captureWindow.rootViewController.view ?: cameraOverlay; + captureRoot.frame = captureWindow.bounds; + + SimDeckPickerCaptureControl *capture = objc_getAssociatedObject(picker, &kPickerCaptureControlKey); + if (!capture) { + capture = [[[SimDeckPickerCaptureControl alloc] initWithPicker:picker] autorelease]; + objc_setAssociatedObject(picker, &kPickerCaptureControlKey, capture, OBJC_ASSOCIATION_RETAIN_NONATOMIC); + [captureRoot addSubview:capture]; + DebugLog(@"installed UIImagePicker capture control"); + } else if (capture.superview != captureRoot) { + [capture removeFromSuperview]; + [captureRoot addSubview:capture]; + } + capture.frame = captureRoot.bounds; + [captureRoot bringSubviewToFront:capture]; +} + +static void HidePickerOverlayWindow(UIImagePickerController *picker) { + UIWindow *captureWindow = objc_getAssociatedObject(picker, &kPickerCaptureWindowKey); + captureWindow.hidden = YES; +} + +@implementation UIViewController (SimDeckCameraPicker) + +- (void)sd_viewDidAppear:(BOOL)animated { + [self sd_viewDidAppear:animated]; + if ([self isKindOfClass:UIImagePickerController.class]) { + InstallPickerOverlay((UIImagePickerController *)self); + } +} + +- (void)sd_viewDidLayoutSubviews { + [self sd_viewDidLayoutSubviews]; + if ([self isKindOfClass:UIImagePickerController.class]) { + InstallPickerOverlay((UIImagePickerController *)self); + } +} + +- (void)sd_viewDidDisappear:(BOOL)animated { + [self sd_viewDidDisappear:animated]; + if ([self isKindOfClass:UIImagePickerController.class]) { + HidePickerOverlayWindow((UIImagePickerController *)self); + } +} + +@end + +@implementation AVCaptureVideoPreviewLayer (SimDeckCamera) + ++ (instancetype)sd_layerWithSession:(AVCaptureSession *)session { + AVCaptureVideoPreviewLayer *layer = [self sd_layerWithSession:session]; + if (OpenSharedCamera()) RegisterPreviewLayer(layer); + return layer; +} + +- (instancetype)sd_initWithSession:(AVCaptureSession *)session { + id layer = [self sd_initWithSession:session]; + if (layer && OpenSharedCamera()) { + RegisterPreviewLayer(layer); + } + return layer; +} + +- (void)sd_setSession:(AVCaptureSession *)session { + [self sd_setSession:session]; + RegisterPreviewLayer(self); +} + +@end + +static void ExchangeInstance(Class cls, SEL original, SEL replacement) { + Method a = class_getInstanceMethod(cls, original); + Method b = class_getInstanceMethod(cls, replacement); + if (!a || !b) { + Log(@"missing instance method %@ on %@", NSStringFromSelector(original), NSStringFromClass(cls)); + return; + } + method_exchangeImplementations(a, b); +} + +static void ExchangeClass(Class cls, SEL original, SEL replacement) { + Method a = class_getClassMethod(cls, original); + Method b = class_getClassMethod(cls, replacement); + if (!a || !b) { + Log(@"missing class method %@ on %@", NSStringFromSelector(original), NSStringFromClass(cls)); + return; + } + method_exchangeImplementations(a, b); +} + +__attribute__((constructor)) +static void SimDeckCameraInstall(void) { + @autoreleasepool { + gSessions = [[NSMutableArray alloc] init]; + gVideoOutputs = [[NSMutableArray alloc] init]; + gPreviewLayers = [[NSHashTable weakObjectsHashTable] retain]; + gHookedVideoOutputClasses = [[NSMutableSet alloc] init]; + OpenSharedCamera(); + + ExchangeClass(AVCaptureDevice.class, @selector(defaultDeviceWithMediaType:), @selector(sd_defaultDeviceWithMediaType:)); + ExchangeClass(AVCaptureDevice.class, @selector(defaultDeviceWithDeviceType:mediaType:position:), @selector(sd_defaultDeviceWithDeviceType:mediaType:position:)); + ExchangeClass(AVCaptureDevice.class, @selector(devicesWithMediaType:), @selector(sd_devicesWithMediaType:)); + ExchangeClass(AVCaptureDevice.class, @selector(authorizationStatusForMediaType:), @selector(sd_authorizationStatusForMediaType:)); + ExchangeClass(AVCaptureDevice.class, @selector(requestAccessForMediaType:completionHandler:), @selector(sd_requestAccessForMediaType:completionHandler:)); + ExchangeInstance(AVCaptureDeviceDiscoverySession.class, @selector(devices), @selector(sd_devices)); + + ExchangeClass(AVCaptureDeviceInput.class, @selector(deviceInputWithDevice:error:), @selector(sd_deviceInputWithDevice:error:)); + ExchangeInstance(AVCaptureDeviceInput.class, @selector(initWithDevice:error:), @selector(sd_initWithDevice:error:)); + ExchangeInstance(AVCaptureDeviceInput.class, @selector(device), @selector(sd_device)); + ExchangeInstance(AVCaptureDeviceInput.class, @selector(ports), @selector(sd_ports)); + + InstallVideoOutputAllocationHook(); + InstallExistingVideoOutputDelegateHooks(); + ExchangeInstance(AVCapturePhotoOutput.class, @selector(capturePhotoWithSettings:delegate:), @selector(sd_capturePhotoWithSettings:delegate:)); + + ExchangeInstance(AVCaptureSession.class, @selector(canAddInput:), @selector(sd_canAddInput:)); + ExchangeInstance(AVCaptureSession.class, @selector(addInput:), @selector(sd_addInput:)); + ExchangeInstance(AVCaptureSession.class, @selector(canAddOutput:), @selector(sd_canAddOutput:)); + ExchangeInstance(AVCaptureSession.class, @selector(addOutput:), @selector(sd_addOutput:)); + ExchangeInstance(AVCaptureSession.class, @selector(inputs), @selector(sd_inputs)); + ExchangeInstance(AVCaptureSession.class, @selector(outputs), @selector(sd_outputs)); + ExchangeInstance(AVCaptureSession.class, @selector(startRunning), @selector(sd_startRunning)); + ExchangeInstance(AVCaptureSession.class, @selector(stopRunning), @selector(sd_stopRunning)); + ExchangeInstance(AVCaptureSession.class, @selector(isRunning), @selector(sd_isRunning)); + + ExchangeInstance(UIViewController.class, @selector(viewDidAppear:), @selector(sd_viewDidAppear:)); + ExchangeInstance(UIViewController.class, @selector(viewDidLayoutSubviews), @selector(sd_viewDidLayoutSubviews)); + ExchangeInstance(UIViewController.class, @selector(viewDidDisappear:), @selector(sd_viewDidDisappear:)); + ExchangeClass(AVCaptureVideoPreviewLayer.class, @selector(layerWithSession:), @selector(sd_layerWithSession:)); + ExchangeInstance(AVCaptureVideoPreviewLayer.class, @selector(initWithSession:), @selector(sd_initWithSession:)); + ExchangeInstance(AVCaptureVideoPreviewLayer.class, @selector(setSession:), @selector(sd_setSession:)); + DebugLog(@"installed"); + } +} diff --git a/cli/camera/SimDeckCameraService.m b/cli/camera/SimDeckCameraService.m new file mode 100644 index 00000000..5cd10376 --- /dev/null +++ b/cli/camera/SimDeckCameraService.m @@ -0,0 +1,800 @@ +#import +#import +#import +#import + +#import "SimDeckCameraShared.h" + +#import +#import +#import +#import +#import +#import +#import +#import +#import +#import +#import +#import +#import + +static uint32_t gWidth = 1280; +static uint32_t gHeight = 720; +static char *gShmName = NULL; +static SimDeckCameraHeader *gHeader = NULL; +static uint8_t *gPixels = NULL; +static size_t gMappedSize = 0; +static dispatch_queue_t gWriteQueue; +static dispatch_source_t gPlaceholderTimer; +static AVCaptureSession *gWebcamSession; +static id gWebcamDelegate; +static atomic_uint gSourceGeneration; +static atomic_ullong gPublishedFrames; +static atomic_ullong gDroppedFrames; +static NSString *gSourceName = nil; +static NSString *gSourceArgument = nil; +static uint32_t gSourceKind = SIMDECK_CAMERA_SOURCE_PLACEHOLDER; +static OSType gLastPixelFormat = 0; +static BOOL gServiceStarted = NO; +static NSString *gActiveUDID = nil; + +static NSObject *CameraLock(void) { + static NSObject *lock; + static dispatch_once_t once; + dispatch_once(&once, ^{ + lock = [NSObject new]; + }); + return lock; +} + +static void RunOnMainSync(dispatch_block_t block) { + if ([NSThread isMainThread]) { + block(); + } else { + dispatch_sync(dispatch_get_main_queue(), block); + } +} + +static void RunMainRunLoopUntil(BOOL (^isFinished)(void), NSTimeInterval timeout) { + NSDate *deadline = [NSDate dateWithTimeIntervalSinceNow:timeout]; + while (!isFinished() && [deadline timeIntervalSinceNow] > 0) { + [[NSRunLoop mainRunLoop] runMode:NSDefaultRunLoopMode + beforeDate:[NSDate dateWithTimeIntervalSinceNow:0.1]]; + } +} + +static uint64_t NowNs(void) { + return (uint64_t)([[NSDate date] timeIntervalSince1970] * 1000000000.0); +} + +static NSString *StringFromCString(const char *value) { + return value ? [NSString stringWithUTF8String:value] ?: @"" : @""; +} + +static NSString *FourCCString(OSType value) { + char chars[5] = { + (char)((value >> 24) & 0xff), + (char)((value >> 16) & 0xff), + (char)((value >> 8) & 0xff), + (char)(value & 0xff), + '\0', + }; + for (NSUInteger index = 0; index < 4; index += 1) { + if (chars[index] < 32 || chars[index] > 126) { + return [NSString stringWithFormat:@"0x%08x", value]; + } + } + return [NSString stringWithUTF8String:chars] ?: [NSString stringWithFormat:@"0x%08x", value]; +} + +static NSString *AuthorizationStatusName(AVAuthorizationStatus status) { + switch (status) { + case AVAuthorizationStatusAuthorized: return @"authorized"; + case AVAuthorizationStatusDenied: return @"denied"; + case AVAuthorizationStatusRestricted: return @"restricted"; + case AVAuthorizationStatusNotDetermined: return @"not-determined"; + } + return @"unknown"; +} + +static BOOL EnsureCameraAccess(NSString **error) { + AVAuthorizationStatus status = [AVCaptureDevice authorizationStatusForMediaType:AVMediaTypeVideo]; + if (status == AVAuthorizationStatusAuthorized) { + return YES; + } + if (status == AVAuthorizationStatusNotDetermined) { + __block BOOL granted = NO; + __block BOOL finished = NO; + void (^requestAccess)(void) = ^{ + [NSApplication sharedApplication]; + [NSApp setActivationPolicy:NSApplicationActivationPolicyRegular]; + [NSApp activateIgnoringOtherApps:YES]; + [AVCaptureDevice requestAccessForMediaType:AVMediaTypeVideo completionHandler:^(BOOL didGrant) { + granted = didGrant; + finished = YES; + }]; + }; + if ([NSThread isMainThread]) { + requestAccess(); + RunMainRunLoopUntil(^BOOL{ + return finished; + }, 60.0); + } else { + dispatch_async(dispatch_get_main_queue(), requestAccess); + NSDate *deadline = [NSDate dateWithTimeIntervalSinceNow:60.0]; + while (!finished && [deadline timeIntervalSinceNow] > 0) { + usleep(50 * 1000); + } + } + if (granted) { + return YES; + } + status = [AVCaptureDevice authorizationStatusForMediaType:AVMediaTypeVideo]; + } + if (error) { + *error = [NSString stringWithFormat:@"Mac camera permission is %@ for SimDeck.", AuthorizationStatusName(status)]; + } + return NO; +} + +static uint32_t SourceKindForName(NSString *name) { + NSString *lower = name.lowercaseString; + if ([lower isEqualToString:@"image"]) return SIMDECK_CAMERA_SOURCE_IMAGE; + if ([lower isEqualToString:@"video"]) return SIMDECK_CAMERA_SOURCE_VIDEO; + if ([lower isEqualToString:@"webcam"]) return SIMDECK_CAMERA_SOURCE_WEBCAM; + return SIMDECK_CAMERA_SOURCE_PLACEHOLDER; +} + +static NSString *SourceNameForKind(uint32_t kind) { + switch (kind) { + case SIMDECK_CAMERA_SOURCE_IMAGE: return @"image"; + case SIMDECK_CAMERA_SOURCE_VIDEO: return @"video"; + case SIMDECK_CAMERA_SOURCE_WEBCAM: return @"webcam"; + default: return @"placeholder"; + } +} + +static NSString *MirrorName(uint32_t mode) { + switch (mode) { + case SIMDECK_CAMERA_MIRROR_ON: return @"on"; + case SIMDECK_CAMERA_MIRROR_OFF: return @"off"; + default: return @"auto"; + } +} + +static uint32_t MirrorModeForName(NSString *name) { + NSString *lower = name.lowercaseString; + if ([lower isEqualToString:@"on"]) return SIMDECK_CAMERA_MIRROR_ON; + if ([lower isEqualToString:@"off"]) return SIMDECK_CAMERA_MIRROR_OFF; + return SIMDECK_CAMERA_MIRROR_AUTO; +} + +static void SetSourceMetadata(uint32_t sourceKind, NSString *argument) { + if (!gHeader) return; + gHeader->sourceKind = sourceKind; + memset(gHeader->sourceLabel, 0, sizeof(gHeader->sourceLabel)); + NSString *label = argument.length > 0 ? argument : SourceNameForKind(sourceKind); + NSData *labelData = [label dataUsingEncoding:NSUTF8StringEncoding]; + if (labelData.length > 0) { + memcpy(gHeader->sourceLabel, + labelData.bytes, + MIN(labelData.length, sizeof(gHeader->sourceLabel) - 1)); + } +} + +static void SetSourceState(uint32_t sourceKind, NSString *name, NSString *argument) { + gSourceKind = sourceKind; + gSourceName = [name copy]; + gSourceArgument = [argument copy]; +} + +static void PublishBGRA(const uint8_t *source, + uint32_t sourceWidth, + uint32_t sourceHeight, + size_t sourceBytesPerRow, + uint32_t sourceKind, + NSString *label) { + if (!gHeader || !gPixels || !source || sourceWidth == 0 || sourceHeight == 0) return; + dispatch_sync(gWriteQueue, ^{ + gHeader->sequence += 1; + for (uint32_t y = 0; y < gHeight; y += 1) { + uint32_t sy = (uint32_t)(((uint64_t)y * sourceHeight) / MAX(gHeight, 1)); + const uint8_t *sourceRow = source + ((size_t)sy * sourceBytesPerRow); + uint8_t *destRow = gPixels + ((size_t)y * gHeader->bytesPerRow); + for (uint32_t x = 0; x < gWidth; x += 1) { + uint32_t sx = (uint32_t)(((uint64_t)x * sourceWidth) / MAX(gWidth, 1)); + const uint8_t *pixel = sourceRow + ((size_t)sx * 4); + uint8_t *out = destRow + ((size_t)x * 4); + out[0] = pixel[0]; + out[1] = pixel[1]; + out[2] = pixel[2]; + out[3] = 0xff; + } + } + gHeader->timestampNs = NowNs(); + gHeader->sourceKind = sourceKind; + SetSourceMetadata(sourceKind, label); + gHeader->sequence += 1; + atomic_fetch_add(&gPublishedFrames, 1); + }); +} + +static void PublishPixelBuffer(CVPixelBufferRef pixelBuffer, uint32_t sourceKind, NSString *label) { + if (!pixelBuffer) return; + OSType format = CVPixelBufferGetPixelFormatType(pixelBuffer); + gLastPixelFormat = format; + if (format == kCVPixelFormatType_32BGRA) { + CVPixelBufferLockBaseAddress(pixelBuffer, kCVPixelBufferLock_ReadOnly); + PublishBGRA((const uint8_t *)CVPixelBufferGetBaseAddress(pixelBuffer), + (uint32_t)CVPixelBufferGetWidth(pixelBuffer), + (uint32_t)CVPixelBufferGetHeight(pixelBuffer), + CVPixelBufferGetBytesPerRow(pixelBuffer), + sourceKind, + label); + CVPixelBufferUnlockBaseAddress(pixelBuffer, kCVPixelBufferLock_ReadOnly); + return; + } + + CIImage *image = [CIImage imageWithCVPixelBuffer:pixelBuffer]; + if (!image) { + atomic_fetch_add(&gDroppedFrames, 1); + return; + } + static CIContext *context; + static dispatch_once_t once; + dispatch_once(&once, ^{ + context = [CIContext contextWithOptions:@{ kCIContextWorkingColorSpace: [NSNull null] }]; + }); + size_t width = CVPixelBufferGetWidth(pixelBuffer); + size_t height = CVPixelBufferGetHeight(pixelBuffer); + size_t bytesPerRow = width * 4; + NSMutableData *data = [NSMutableData dataWithLength:bytesPerRow * height]; + CGColorSpaceRef colorSpace = CGColorSpaceCreateDeviceRGB(); + [context render:image + toBitmap:data.mutableBytes + rowBytes:bytesPerRow + bounds:CGRectMake(0, 0, width, height) + format:kCIFormatBGRA8 + colorSpace:colorSpace]; + CGColorSpaceRelease(colorSpace); + PublishBGRA(data.bytes, + (uint32_t)width, + (uint32_t)height, + bytesPerRow, + sourceKind, + label); +} + +static void DrawPlaceholderFrame(uint32_t frameIndex) { + if (!gHeader || !gPixels) return; + dispatch_sync(gWriteQueue, ^{ + gHeader->sequence += 1; + for (uint32_t y = 0; y < gHeight; y += 1) { + uint8_t *row = gPixels + ((size_t)y * gHeader->bytesPerRow); + for (uint32_t x = 0; x < gWidth; x += 1) { + uint8_t *p = row + ((size_t)x * 4); + uint8_t stripe = (uint8_t)(((x / 80) + (frameIndex / 6)) % 2 ? 56 : 24); + p[0] = (uint8_t)((x + frameIndex * 7) % 256); + p[1] = (uint8_t)((y + frameIndex * 3) % 256); + p[2] = (uint8_t)(180 + stripe); + p[3] = 0xff; + } + } + gHeader->timestampNs = NowNs(); + SetSourceMetadata(SIMDECK_CAMERA_SOURCE_PLACEHOLDER, @"placeholder"); + gHeader->sequence += 1; + atomic_fetch_add(&gPublishedFrames, 1); + }); +} + +static BOOL PublishImageAtPath(NSString *path, NSString **error) { + NSImage *image = [[NSImage alloc] initWithContentsOfFile:path]; + CGImageRef cgImage = [image CGImageForProposedRect:NULL context:nil hints:nil]; + if (!cgImage) { + if (error) *error = [NSString stringWithFormat:@"Unable to decode image at %@", path]; + return NO; + } + size_t sourceWidth = CGImageGetWidth(cgImage); + size_t sourceHeight = CGImageGetHeight(cgImage); + size_t bytesPerRow = sourceWidth * 4; + NSMutableData *data = [NSMutableData dataWithLength:bytesPerRow * sourceHeight]; + CGColorSpaceRef colorSpace = CGColorSpaceCreateDeviceRGB(); + CGContextRef context = CGBitmapContextCreate(data.mutableBytes, + sourceWidth, + sourceHeight, + 8, + bytesPerRow, + colorSpace, + kCGBitmapByteOrder32Little | kCGImageAlphaPremultipliedFirst); + CGColorSpaceRelease(colorSpace); + if (!context) { + if (error) *error = @"Unable to allocate image conversion buffer."; + return NO; + } + CGContextDrawImage(context, CGRectMake(0, 0, sourceWidth, sourceHeight), cgImage); + CGContextRelease(context); + PublishBGRA(data.bytes, + (uint32_t)sourceWidth, + (uint32_t)sourceHeight, + bytesPerRow, + SIMDECK_CAMERA_SOURCE_IMAGE, + path); + return YES; +} + +@interface SimDeckCameraWebcamWriter : NSObject +@property (nonatomic, copy) NSString *label; +@end + +@implementation SimDeckCameraWebcamWriter + +- (void)captureOutput:(AVCaptureOutput *)output + didOutputSampleBuffer:(CMSampleBufferRef)sampleBuffer + fromConnection:(AVCaptureConnection *)connection { + (void)output; + (void)connection; + PublishPixelBuffer(CMSampleBufferGetImageBuffer(sampleBuffer), + SIMDECK_CAMERA_SOURCE_WEBCAM, + self.label); +} + +@end + +static NSArray *CameraDevices(void) { + AVCaptureDeviceDiscoverySession *session = [AVCaptureDeviceDiscoverySession + discoverySessionWithDeviceTypes:@[ + AVCaptureDeviceTypeBuiltInWideAngleCamera, + AVCaptureDeviceTypeExternal, + AVCaptureDeviceTypeContinuityCamera, + ] + mediaType:AVMediaTypeVideo + position:AVCaptureDevicePositionUnspecified]; + return session.devices ?: @[]; +} + +static AVCaptureDevice *PickCamera(NSString *wanted) { + NSArray *devices = CameraDevices(); + if (wanted.length == 0) { + for (AVCaptureDevice *device in devices) { + if (device.position == AVCaptureDevicePositionFront) return device; + } + return devices.firstObject; + } + NSString *needle = wanted.lowercaseString; + for (AVCaptureDevice *device in devices) { + if ([device.uniqueID.lowercaseString isEqualToString:needle]) return device; + } + for (AVCaptureDevice *device in devices) { + if ([device.localizedName.lowercaseString containsString:needle]) return device; + } + return nil; +} + +static void StopCurrentSource(void) { + atomic_fetch_add(&gSourceGeneration, 1); + if (gPlaceholderTimer) { + dispatch_source_cancel(gPlaceholderTimer); + gPlaceholderTimer = nil; + } + if (gWebcamSession) { + [gWebcamSession stopRunning]; + gWebcamSession = nil; + } + gWebcamDelegate = nil; +} + +static BOOL StartPlaceholder(NSString **error) { + (void)error; + StopCurrentSource(); + SetSourceState(SIMDECK_CAMERA_SOURCE_PLACEHOLDER, @"placeholder", nil); + __block uint32_t frame = 0; + DrawPlaceholderFrame(frame); + dispatch_queue_t queue = dispatch_queue_create("dev.nativescript.simdeck.camera.placeholder", DISPATCH_QUEUE_SERIAL); + gPlaceholderTimer = dispatch_source_create(DISPATCH_SOURCE_TYPE_TIMER, 0, 0, queue); + dispatch_source_set_timer(gPlaceholderTimer, + dispatch_time(DISPATCH_TIME_NOW, 0), + (uint64_t)(NSEC_PER_SEC / 30), + (uint64_t)(NSEC_PER_MSEC * 5)); + dispatch_source_set_event_handler(gPlaceholderTimer, ^{ + DrawPlaceholderFrame(frame++); + }); + dispatch_resume(gPlaceholderTimer); + return YES; +} + +static BOOL StartImage(NSString *path, NSString **error) { + StopCurrentSource(); + if (!PublishImageAtPath(path, error)) { + return NO; + } + SetSourceState(SIMDECK_CAMERA_SOURCE_IMAGE, @"image", path); + return YES; +} + +static BOOL StartVideo(NSString *path, NSString **error) { + NSURL *url = nil; + NSURLComponents *components = [NSURLComponents componentsWithString:path ?: @""]; + if (components.scheme.length > 0) { + url = components.URL; + } else if ([[NSFileManager defaultManager] fileExistsAtPath:path]) { + url = [NSURL fileURLWithPath:path]; + } + if (!url) { + if (error) *error = [NSString stringWithFormat:@"Video file does not exist: %@", path]; + return NO; + } + StopCurrentSource(); + SetSourceState(SIMDECK_CAMERA_SOURCE_VIDEO, @"video", path); + unsigned generation = atomic_load(&gSourceGeneration); + dispatch_async(dispatch_get_global_queue(QOS_CLASS_USER_INITIATED, 0), ^{ + while (atomic_load(&gSourceGeneration) == generation) { + @autoreleasepool { + AVAsset *asset = [AVAsset assetWithURL:url]; +#pragma clang diagnostic push +#pragma clang diagnostic ignored "-Wdeprecated-declarations" + NSArray *tracks = [asset tracksWithMediaType:AVMediaTypeVideo]; +#pragma clang diagnostic pop + AVAssetTrack *track = tracks.firstObject; + if (!track) { + usleep(300 * 1000); + continue; + } + NSError *readerError = nil; + AVAssetReader *reader = [[AVAssetReader alloc] initWithAsset:asset error:&readerError]; + if (!reader) { + fprintf(stderr, "simdeck-camera: video reader failed: %s\n", readerError.localizedDescription.UTF8String); + usleep(300 * 1000); + continue; + } + NSDictionary *settings = @{ + (id)kCVPixelBufferPixelFormatTypeKey: @(kCVPixelFormatType_32BGRA), + }; + AVAssetReaderTrackOutput *output = [[AVAssetReaderTrackOutput alloc] initWithTrack:track outputSettings:settings]; + output.alwaysCopiesSampleData = NO; + if (![reader canAddOutput:output]) { + usleep(300 * 1000); + continue; + } + [reader addOutput:output]; + if (![reader startReading]) { + usleep(300 * 1000); + continue; + } + while (atomic_load(&gSourceGeneration) == generation && reader.status == AVAssetReaderStatusReading) { + CMSampleBufferRef sample = [output copyNextSampleBuffer]; + if (!sample) break; + PublishPixelBuffer(CMSampleBufferGetImageBuffer(sample), SIMDECK_CAMERA_SOURCE_VIDEO, path); + CFRelease(sample); + usleep(33333); + } + } + } + }); + return YES; +} + +static BOOL StartWebcam(NSString *requestedDevice, NSString **error) { + StopCurrentSource(); + if (!EnsureCameraAccess(error)) { + return NO; + } + AVCaptureDevice *device = PickCamera(requestedDevice); + if (!device) { + if (error) *error = requestedDevice.length > 0 + ? [NSString stringWithFormat:@"No matching Mac camera: %@", requestedDevice] + : @"No Mac camera is available."; + return NO; + } + NSError *inputError = nil; + AVCaptureDeviceInput *input = [AVCaptureDeviceInput deviceInputWithDevice:device error:&inputError]; + if (!input) { + if (error) *error = inputError.localizedDescription ?: @"Unable to open Mac camera."; + return NO; + } + AVCaptureSession *session = [[AVCaptureSession alloc] init]; + session.sessionPreset = AVCaptureSessionPreset1280x720; + if (![session canAddInput:input]) { + if (error) *error = @"Unable to attach Mac camera input."; + return NO; + } + [session addInput:input]; + AVCaptureVideoDataOutput *output = [[AVCaptureVideoDataOutput alloc] init]; + output.videoSettings = @{ (id)kCVPixelBufferPixelFormatTypeKey: @(kCVPixelFormatType_32BGRA) }; + output.alwaysDiscardsLateVideoFrames = YES; + SimDeckCameraWebcamWriter *writer = [[SimDeckCameraWebcamWriter alloc] init]; + writer.label = device.localizedName ?: @"webcam"; + [output setSampleBufferDelegate:writer queue:dispatch_queue_create("dev.nativescript.simdeck.camera.webcam", DISPATCH_QUEUE_SERIAL)]; + if (![session canAddOutput:output]) { + if (error) *error = @"Unable to attach Mac camera output."; + return NO; + } + [session addOutput:output]; + gWebcamSession = session; + gWebcamDelegate = writer; + [session startRunning]; + if (!session.isRunning) { + if (error) *error = @"Mac camera session did not start."; + StopCurrentSource(); + return NO; + } + uint64_t startFrames = atomic_load(&gPublishedFrames); + NSDate *deadline = [NSDate dateWithTimeIntervalSinceNow:4.0]; + while (atomic_load(&gPublishedFrames) == startFrames && [deadline timeIntervalSinceNow] > 0) { + [[NSRunLoop mainRunLoop] runMode:NSDefaultRunLoopMode beforeDate:[NSDate dateWithTimeIntervalSinceNow:0.05]]; + usleep(20 * 1000); + } + if (atomic_load(&gPublishedFrames) == startFrames) { + if (error) { + *error = [NSString stringWithFormat:@"Mac camera session started but delivered no frames. Authorization=%@ pixelFormat=%@", + AuthorizationStatusName([AVCaptureDevice authorizationStatusForMediaType:AVMediaTypeVideo]), + gLastPixelFormat ? FourCCString(gLastPixelFormat) : @"none"]; + } + StopCurrentSource(); + return NO; + } + SetSourceState(SIMDECK_CAMERA_SOURCE_WEBCAM, @"webcam", device.uniqueID ?: device.localizedName); + return YES; +} + +static BOOL SwitchSource(NSString *source, NSString *argument, NSString **error) { + uint32_t kind = SourceKindForName(source); + switch (kind) { + case SIMDECK_CAMERA_SOURCE_IMAGE: + return StartImage(argument ?: @"", error); + case SIMDECK_CAMERA_SOURCE_VIDEO: + return StartVideo(argument ?: @"", error); + case SIMDECK_CAMERA_SOURCE_WEBCAM: + return StartWebcam(argument ?: @"", error); + default: + return StartPlaceholder(error); + } +} + +static NSDictionary *StatusPayload(BOOL ok, NSString *error) { + NSMutableDictionary *payload = [@{ + @"ok": @(ok), + @"alive": @YES, + @"source": gSourceName ?: SourceNameForKind(gSourceKind), + @"mirror": gHeader ? MirrorName(gHeader->mirrorMode) : @"auto", + @"width": @(gWidth), + @"height": @(gHeight), + @"processId": @((int)getpid()), + @"sequence": gHeader ? @(gHeader->sequence) : @0, + @"frames": @(atomic_load(&gPublishedFrames)), + @"droppedFrames": @(atomic_load(&gDroppedFrames)), + } mutableCopy]; + if (gSourceArgument.length > 0) payload[@"arg"] = gSourceArgument; + if (gHeader) payload[@"sourceLabel"] = StringFromCString(gHeader->sourceLabel); + if (gLastPixelFormat != 0) payload[@"pixelFormat"] = FourCCString(gLastPixelFormat); + payload[@"cameraAuthorization"] = AuthorizationStatusName([AVCaptureDevice authorizationStatusForMediaType:AVMediaTypeVideo]); + if (gSourceKind == SIMDECK_CAMERA_SOURCE_WEBCAM) { + payload[@"webcamRunning"] = @(gWebcamSession.isRunning); + } + if (error.length > 0) payload[@"error"] = error; + return payload; +} + +static int OpenSharedMemory(void) { + if (!gShmName) return -1; + shm_unlink(gShmName); + gMappedSize = (size_t)SimDeckCameraBufferSize(gWidth, gHeight); + int fd = shm_open(gShmName, O_CREAT | O_RDWR, 0644); + if (fd < 0) { + perror("shm_open"); + return -1; + } + if (ftruncate(fd, (off_t)gMappedSize) != 0) { + perror("ftruncate"); + close(fd); + return -1; + } + void *mapped = mmap(NULL, gMappedSize, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0); + close(fd); + if (mapped == MAP_FAILED) { + perror("mmap"); + return -1; + } + gHeader = (SimDeckCameraHeader *)mapped; + memset(gHeader, 0, SIMDECK_CAMERA_HEADER_SIZE); + gHeader->magic = SIMDECK_CAMERA_MAGIC; + gHeader->version = SIMDECK_CAMERA_VERSION; + gHeader->headerSize = SIMDECK_CAMERA_HEADER_SIZE; + gHeader->width = gWidth; + gHeader->height = gHeight; + gHeader->bytesPerRow = gWidth * 4; + gHeader->pixelFormat = kCVPixelFormatType_32BGRA; + gHeader->mirrorMode = SIMDECK_CAMERA_MIRROR_AUTO; + gPixels = ((uint8_t *)mapped) + SIMDECK_CAMERA_HEADER_SIZE; + return 0; +} + +static NSDictionary *WebcamsPayload(void) { + NSMutableArray *items = [NSMutableArray array]; + for (AVCaptureDevice *device in CameraDevices()) { + [items addObject:@{ + @"id": device.uniqueID ?: device.localizedName ?: @"", + @"name": device.localizedName ?: device.uniqueID ?: @"Camera", + @"position": device.position == AVCaptureDevicePositionFront ? @"front" : + device.position == AVCaptureDevicePositionBack ? @"back" : @"unspecified", + }]; + } + return @{ @"webcams": items }; +} + +static void Cleanup(void) { + StopCurrentSource(); + if (gHeader) { + munmap(gHeader, gMappedSize); + gHeader = NULL; + } + if (gShmName) { + shm_unlink(gShmName); + free(gShmName); + gShmName = NULL; + } + gPixels = NULL; + gMappedSize = 0; + gSourceName = nil; + gSourceArgument = nil; + gSourceKind = SIMDECK_CAMERA_SOURCE_PLACEHOLDER; + gActiveUDID = nil; + gServiceStarted = NO; +} + +static void SignalHandler(int signalNumber) { + (void)signalNumber; + Cleanup(); + _exit(0); +} + +static char *CopyCString(NSString *value) { + const char *utf8 = value.UTF8String ?: ""; + char *copy = strdup(utf8); + return copy ?: strdup(""); +} + +static void SetNativeError(char **errorMessage, NSString *message) { + if (errorMessage) { + *errorMessage = CopyCString(message ?: @"Unknown camera error."); + } +} + +static char *JSONCString(NSDictionary *payload) { + NSData *data = [NSJSONSerialization dataWithJSONObject:payload ?: @{} options:0 error:nil]; + if (!data) { + return CopyCString(@"{}"); + } + NSString *json = [[NSString alloc] initWithData:data encoding:NSUTF8StringEncoding] ?: @"{}"; + return CopyCString(json); +} + +char *simdeck_camera_list_webcams_json(char **errorMessage) { + (void)errorMessage; + __block char *result = NULL; + RunOnMainSync(^{ + @autoreleasepool { + result = JSONCString(WebcamsPayload()); + } + }); + return result; +} + +bool simdeck_camera_start(const char *udid, + const char *shmName, + const char *source, + const char *sourceArgument, + const char *mirror, + char **errorMessage) { + __block BOOL ok = NO; + __block NSString *nativeError = nil; + RunOnMainSync(^{ + @autoreleasepool { + @synchronized (CameraLock()) { + Cleanup(); + if (!shmName || shmName[0] != '/') { + nativeError = @"Camera shared memory name must start with `/`."; + return; + } + gActiveUDID = [StringFromCString(udid) copy]; + gShmName = strdup(shmName); + gWriteQueue = dispatch_queue_create("dev.nativescript.simdeck.camera.writer", DISPATCH_QUEUE_SERIAL); + atomic_store(&gPublishedFrames, 0); + atomic_store(&gDroppedFrames, 0); + gLastPixelFormat = 0; + [NSApplication sharedApplication]; + [NSApp finishLaunching]; + signal(SIGINT, SignalHandler); + signal(SIGTERM, SignalHandler); + if (OpenSharedMemory() != 0) { + nativeError = @"Unable to open camera shared memory."; + Cleanup(); + return; + } + if (gHeader) { + gHeader->mirrorMode = MirrorModeForName(StringFromCString(mirror)); + } + if (!SwitchSource(StringFromCString(source), StringFromCString(sourceArgument), &nativeError)) { + Cleanup(); + return; + } + gServiceStarted = YES; + ok = YES; + } + } + }); + if (!ok) { + SetNativeError(errorMessage, nativeError); + } + return ok; +} + +char *simdeck_camera_status(const char *udid, char **errorMessage) { + (void)errorMessage; + __block char *result = NULL; + RunOnMainSync(^{ + @autoreleasepool { + @synchronized (CameraLock()) { + NSString *requestedUDID = StringFromCString(udid); + if (!gServiceStarted || (requestedUDID.length > 0 && gActiveUDID.length > 0 && ![requestedUDID isEqualToString:gActiveUDID])) { + result = JSONCString(@{ + @"ok": @YES, + @"alive": @NO, + @"cameraAuthorization": AuthorizationStatusName([AVCaptureDevice authorizationStatusForMediaType:AVMediaTypeVideo]), + }); + return; + } + result = JSONCString(StatusPayload(YES, nil)); + } + } + }); + return result; +} + +char *simdeck_camera_switch(const char *udid, + const char *source, + const char *sourceArgument, + const char *mirror, + char **errorMessage) { + __block char *result = NULL; + __block NSString *nativeError = nil; + RunOnMainSync(^{ + @autoreleasepool { + @synchronized (CameraLock()) { + NSString *requestedUDID = StringFromCString(udid); + if (!gServiceStarted || (requestedUDID.length > 0 && gActiveUDID.length > 0 && ![requestedUDID isEqualToString:gActiveUDID])) { + nativeError = @"Camera simulation is not running for this simulator."; + return; + } + if (mirror && mirror[0] && gHeader) { + gHeader->mirrorMode = MirrorModeForName(StringFromCString(mirror)); + } + if (source && source[0] && !SwitchSource(StringFromCString(source), StringFromCString(sourceArgument), &nativeError)) { + return; + } + result = JSONCString(StatusPayload(YES, nil)); + } + } + }); + if (!result) { + SetNativeError(errorMessage, nativeError); + } + return result; +} + +bool simdeck_camera_stop(const char *udid, char **errorMessage) { + (void)errorMessage; + __block BOOL stopped = NO; + RunOnMainSync(^{ + @autoreleasepool { + @synchronized (CameraLock()) { + NSString *requestedUDID = StringFromCString(udid); + if (requestedUDID.length == 0 || gActiveUDID.length == 0 || [requestedUDID isEqualToString:gActiveUDID]) { + Cleanup(); + } + stopped = YES; + } + } + }); + return stopped; +} diff --git a/cli/camera/SimDeckCameraShared.h b/cli/camera/SimDeckCameraShared.h new file mode 100644 index 00000000..e86b7810 --- /dev/null +++ b/cli/camera/SimDeckCameraShared.h @@ -0,0 +1,34 @@ +#pragma once + +#include + +#define SIMDECK_CAMERA_MAGIC 0x4d434453u +#define SIMDECK_CAMERA_VERSION 1u +#define SIMDECK_CAMERA_HEADER_SIZE 4096u +#define SIMDECK_CAMERA_SOURCE_PLACEHOLDER 1u +#define SIMDECK_CAMERA_SOURCE_IMAGE 2u +#define SIMDECK_CAMERA_SOURCE_VIDEO 3u +#define SIMDECK_CAMERA_SOURCE_WEBCAM 4u +#define SIMDECK_CAMERA_MIRROR_AUTO 0u +#define SIMDECK_CAMERA_MIRROR_OFF 1u +#define SIMDECK_CAMERA_MIRROR_ON 2u + +typedef struct SimDeckCameraHeader { + uint32_t magic; + uint32_t version; + uint32_t headerSize; + uint32_t width; + uint32_t height; + uint32_t bytesPerRow; + uint32_t pixelFormat; + uint32_t sourceKind; + volatile uint64_t sequence; + uint64_t timestampNs; + uint32_t mirrorMode; + uint32_t reserved; + char sourceLabel[240]; +} SimDeckCameraHeader; + +static inline uint64_t SimDeckCameraBufferSize(uint32_t width, uint32_t height) { + return (uint64_t)SIMDECK_CAMERA_HEADER_SIZE + ((uint64_t)width * 4u * (uint64_t)height); +} diff --git a/cli/camera/build-injector.sh b/cli/camera/build-injector.sh new file mode 100755 index 00000000..5a2a06c9 --- /dev/null +++ b/cli/camera/build-injector.sh @@ -0,0 +1,38 @@ +#!/usr/bin/env bash +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +OUT_DIR="${1:-"$SCRIPT_DIR/../../build/camera"}" +mkdir -p "$OUT_DIR" + +SDK="$(xcrun --sdk iphonesimulator --show-sdk-path)" +OUT="$OUT_DIR/libSimDeckCameraInjector.dylib" +ARCHES=() +HOST_ARCH="$(uname -m)" +if [[ "$HOST_ARCH" == "arm64" ]]; then + ARCHES=(-arch arm64) +else + ARCHES=(-arch x86_64) +fi + +xcrun --sdk iphonesimulator clang \ + "${ARCHES[@]}" \ + -dynamiclib \ + -fmodules \ + -isysroot "$SDK" \ + -mios-simulator-version-min=15.0 \ + -Wall \ + -Wextra \ + -framework Foundation \ + -framework AVFoundation \ + -framework CoreGraphics \ + -framework CoreMedia \ + -framework CoreVideo \ + -framework QuartzCore \ + -framework UIKit \ + -install_name "@rpath/libSimDeckCameraInjector.dylib" \ + "$SCRIPT_DIR/SimDeckCameraInjector.m" \ + -o "$OUT" + +codesign --force --sign - "$OUT" >/dev/null 2>&1 || true +echo "$OUT" diff --git a/client/src/api/simulators.ts b/client/src/api/simulators.ts index 98cd4fbf..1e31e932 100644 --- a/client/src/api/simulators.ts +++ b/client/src/api/simulators.ts @@ -2,6 +2,9 @@ import { apiRequest } from "./client"; import type { AccessibilitySourcePreference, AccessibilityTreeResponse, + CameraStartRequest, + CameraStatusResponse, + CameraWebcamsResponse, ChromeDevToolsTargetDiscovery, ChromeProfile, CreateSimulatorRequest, @@ -175,6 +178,57 @@ export async function sampleSimulatorProcess( ); } +export async function fetchCameraWebcams( + options: RequestInit = {}, +): Promise { + return apiRequest("/api/camera/webcams", options); +} + +export async function fetchCameraStatus( + udid: string, + options: RequestInit = {}, +): Promise { + return apiRequest( + `/api/simulators/${encodeURIComponent(udid)}/camera`, + options, + ); +} + +export async function startCameraSimulation( + udid: string, + payload: CameraStartRequest, +): Promise { + return apiRequest( + `/api/simulators/${encodeURIComponent(udid)}/camera`, + { + body: JSON.stringify(payload), + method: "POST", + }, + ); +} + +export async function switchCameraSimulationSource( + udid: string, + payload: Pick, +): Promise { + return apiRequest( + `/api/simulators/${encodeURIComponent(udid)}/camera/source`, + { + body: JSON.stringify(payload), + method: "POST", + }, + ); +} + +export async function stopCameraSimulation( + udid: string, +): Promise { + return apiRequest( + `/api/simulators/${encodeURIComponent(udid)}/camera`, + { method: "DELETE" }, + ); +} + export async function fetchWebKitTargets( udid: string, options: RequestInit = {}, diff --git a/client/src/api/types.ts b/client/src/api/types.ts index 1a650054..94cde4dd 100644 --- a/client/src/api/types.ts +++ b/client/src/api/types.ts @@ -206,6 +206,46 @@ export interface InstallUploadResponse { udid: string; } +export type CameraSourceKind = "placeholder" | "image" | "video" | "webcam"; + +export interface CameraSourceRequest { + kind: CameraSourceKind; + arg?: string; +} + +export interface CameraStartRequest { + bundleId?: string; + source: CameraSourceRequest; + mirror?: "auto" | "on" | "off"; +} + +export interface CameraWebcam { + id: string; + name: string; + position?: string; +} + +export interface CameraWebcamsResponse { + webcams: CameraWebcam[]; +} + +export interface CameraStatusResponse { + ok?: boolean; + udid?: string; + alive: boolean; + source?: CameraSourceKind | string; + arg?: string; + sourceLabel?: string; + mirror?: "auto" | "on" | "off" | string; + daemonPid?: number; + bundleIds?: string[]; + width?: number; + height?: number; + sequence?: number; + appLogPath?: string; + error?: string; +} + export interface SimulatorForegroundApp { appName?: string | null; bundleIdentifier?: string | null; diff --git a/client/src/app/AppShell.tsx b/client/src/app/AppShell.tsx index 5f4aff98..b461b639 100644 --- a/client/src/app/AppShell.tsx +++ b/client/src/app/AppShell.tsx @@ -75,6 +75,7 @@ import type { ViewMode, } from "../features/viewport/types"; import { useViewportLayout } from "../features/viewport/useViewportLayout"; +import { CameraSimulationModal } from "../features/simulators/CameraSimulationModal"; import { NewSimulatorModal } from "../features/simulators/NewSimulatorModal"; import { buildShellRotationTransform, @@ -396,6 +397,7 @@ export function AppShell({ ); const [menuOpen, setMenuOpen] = useState(false); const [newSimulatorOpen, setNewSimulatorOpen] = useState(false); + const [cameraSimulationOpen, setCameraSimulationOpen] = useState(false); const [localError, setLocalError] = useState(""); const [failedStreamUDIDs, setFailedStreamUDIDs] = useState>( () => new Set(), @@ -2503,6 +2505,10 @@ export function AppShell({ } }} onInstallAppPrompt={openInstallAppPicker} + onOpenCameraSimulation={() => { + setMenuOpen(false); + setCameraSimulationOpen(true); + }} onOpenAppSwitcher={() => { if (!selectedSimulator) { return; @@ -2611,6 +2617,14 @@ export function AppShell({ open={newSimulatorOpen && !hideSimulatorSelection} selectedSimulator={selectedSimulator} /> + setCameraSimulationOpen(false)} + open={cameraSimulationOpen} + selectedSimulator={selectedSimulator} + /> void; + open: boolean; + selectedSimulator: SimulatorMetadata | null; +} + +type SourceMode = "placeholder" | "webcam" | "media"; +type MirrorMode = "auto" | "on" | "off"; + +export function CameraSimulationModal({ + foregroundBundleId, + onClose, + open, + selectedSimulator, +}: CameraSimulationModalProps) { + const [bundleId, setBundleId] = useState(""); + const [sourceMode, setSourceMode] = useState("placeholder"); + const [mediaPath, setMediaPath] = useState(""); + const [webcamId, setWebcamId] = useState(""); + const [mirror, setMirror] = useState("auto"); + const [status, setStatus] = useState(null); + const [webcams, setWebcams] = useState([]); + const [isLoading, setIsLoading] = useState(false); + const [isApplying, setIsApplying] = useState(false); + const [isStopping, setIsStopping] = useState(false); + const [error, setError] = useState(""); + + const udid = selectedSimulator?.udid ?? ""; + const canApply = Boolean( + selectedSimulator?.isBooted && + bundleId.trim() && + (sourceMode !== "media" || mediaPath.trim()), + ); + + useEffect(() => { + if (!open) { + return; + } + setBundleId(foregroundBundleId ?? ""); + setError(""); + setIsApplying(false); + setIsStopping(false); + void refreshStatus(); + void refreshWebcams(); + }, [foregroundBundleId, open, udid]); + + useEffect(() => { + if (!open) { + return; + } + function handleKeyDown(event: KeyboardEvent) { + if (event.key === "Escape") { + onClose(); + } + } + window.addEventListener("keydown", handleKeyDown); + return () => window.removeEventListener("keydown", handleKeyDown); + }, [onClose, open]); + + const activeBundleText = useMemo(() => { + const bundleIds = status?.bundleIds ?? []; + if (bundleIds.length === 0) { + return status?.alive ? "helper running" : "not running"; + } + return bundleIds.join(", "); + }, [status]); + + if (!open) { + return null; + } + + async function refreshStatus() { + if (!udid) { + setStatus(null); + return; + } + setIsLoading(true); + try { + const nextStatus = await fetchCameraStatus(udid); + setStatus(nextStatus); + if (nextStatus.mirror === "on" || nextStatus.mirror === "off") { + setMirror(nextStatus.mirror); + } + if ( + nextStatus.source === "webcam" || + nextStatus.source === "placeholder" + ) { + setSourceMode(nextStatus.source); + } else if ( + nextStatus.source === "image" || + nextStatus.source === "video" + ) { + setSourceMode("media"); + } + if (nextStatus.arg) { + if (nextStatus.source === "webcam") { + setWebcamId(nextStatus.arg); + } else if ( + nextStatus.source === "image" || + nextStatus.source === "video" + ) { + setMediaPath(nextStatus.arg); + } + } + } catch (statusError) { + setStatus(null); + setError( + statusError instanceof Error + ? statusError.message + : "Unable to load camera status.", + ); + } finally { + setIsLoading(false); + } + } + + async function refreshWebcams() { + try { + const response = await fetchCameraWebcams(); + setWebcams(response.webcams ?? []); + setWebcamId((current) => current || response.webcams?.[0]?.id || ""); + } catch { + setWebcams([]); + } + } + + function requestSource(): { kind: CameraSourceKind; arg?: string } { + if (sourceMode === "webcam") { + return { kind: "webcam", arg: webcamId || undefined }; + } + if (sourceMode === "media") { + const value = mediaPath.trim(); + const kind: CameraSourceKind = looksLikeVideo(value) ? "video" : "image"; + return { kind, arg: value }; + } + return { kind: "placeholder" }; + } + + async function apply(event: FormEvent) { + event.preventDefault(); + if (!selectedSimulator?.isBooted) { + setError( + "Boot the selected simulator before enabling camera simulation.", + ); + return; + } + if (!bundleId.trim()) { + setError( + "Enter the app bundle identifier to relaunch with camera simulation.", + ); + return; + } + setIsApplying(true); + setError(""); + try { + const nextStatus = await startCameraSimulation(udid, { + bundleId: bundleId.trim(), + mirror, + source: requestSource(), + }); + setStatus(nextStatus); + } catch (applyError) { + setError( + applyError instanceof Error + ? applyError.message + : "Unable to start camera simulation.", + ); + } finally { + setIsApplying(false); + } + } + + async function switchSourceOnly() { + if (!status?.alive) { + return; + } + setIsApplying(true); + setError(""); + try { + const nextStatus = await switchCameraSimulationSource(udid, { + mirror, + source: requestSource(), + }); + setStatus(nextStatus); + } catch (switchError) { + setError( + switchError instanceof Error + ? switchError.message + : "Unable to switch camera source.", + ); + } finally { + setIsApplying(false); + } + } + + async function stop() { + setIsStopping(true); + setError(""); + try { + const nextStatus = await stopCameraSimulation(udid); + setStatus(nextStatus); + } catch (stopError) { + setError( + stopError instanceof Error + ? stopError.message + : "Unable to stop camera simulation.", + ); + } finally { + setIsStopping(false); + } + } + + return ( +
{ + if (event.target === event.currentTarget) { + onClose(); + } + }} + > +
+
+
+ +
+
+ + + +
+
+ + {sourceMode === "media" ? ( + + ) : null} + {sourceMode === "webcam" ? ( + + ) : null} + +

+ {isLoading + ? "Loading camera status..." + : `Status: ${activeBundleText}`} +

+
+ {error ?

{error}

: null} +
+ +
+ + + + + +
+
+
+ ); +} + +function looksLikeVideo(value: string): boolean { + if (/^https?:\/\//i.test(value)) { + return true; + } + return /\.(mp4|m4v|mov|qt|avi|mkv|webm|mpg|mpeg|3gp|3g2)$/i.test(value); +} diff --git a/client/src/features/simulators/SimulatorMenu.tsx b/client/src/features/simulators/SimulatorMenu.tsx index fe739271..385e232d 100644 --- a/client/src/features/simulators/SimulatorMenu.tsx +++ b/client/src/features/simulators/SimulatorMenu.tsx @@ -28,6 +28,7 @@ interface SimulatorMenuProps { onDismissKeyboard: () => void; onHome: () => void; onInstallAppPrompt: () => void; + onOpenCameraSimulation: () => void; onOpenAppSwitcher: () => void; onOpenBundlePrompt: () => void; onOpenNewSimulator: () => void; @@ -67,6 +68,7 @@ export function SimulatorMenu({ onDismissKeyboard, onHome, onInstallAppPrompt, + onOpenCameraSimulation, onOpenAppSwitcher, onOpenBundlePrompt, onOpenNewSimulator, @@ -280,6 +282,15 @@ export function SimulatorMenu({ +