diff --git a/sources/vphone-cli/VPhoneAppDelegate.swift b/sources/vphone-cli/VPhoneAppDelegate.swift index 34f6549..5a1ac34 100644 --- a/sources/vphone-cli/VPhoneAppDelegate.swift +++ b/sources/vphone-cli/VPhoneAppDelegate.swift @@ -13,6 +13,7 @@ class VPhoneAppDelegate: NSObject, NSApplicationDelegate { private var appWindowController: VPhoneAppWindowController? private var locationProvider: VPhoneLocationProvider? private var hostControl: VPhoneHostControl? + private var cameraServer: VPhoneCameraServer? private var sigintSource: DispatchSourceSignal? private var didAttemptAutoInstall = false @@ -88,8 +89,12 @@ class VPhoneAppDelegate: NSObject, NSApplicationDelegate { let provider = VPhoneLocationProvider(control: control) locationProvider = provider + let camServer = VPhoneCameraServer() + cameraServer = camServer + if let device = vm.virtualMachine.socketDevices.first as? VZVirtioSocketDevice { control.connect(device: device) + camServer.connect(device: device) } } @@ -135,6 +140,14 @@ class VPhoneAppDelegate: NSObject, NSApplicationDelegate { if let provider = locationProvider { mc.locationProvider = provider } + if let camServer = cameraServer { + mc.cameraServer = camServer + camServer.onConnectionStateChange = { [weak mc] connected in + Task { @MainActor in + mc?.updateCameraConnectionState(connected: connected) + } + } + } let recorder = VPhoneScreenRecorder() mc.screenRecorder = recorder menuController = mc diff --git a/sources/vphone-cli/VPhoneCameraServer.swift b/sources/vphone-cli/VPhoneCameraServer.swift new file mode 100644 index 0000000..d84e4f2 --- /dev/null +++ b/sources/vphone-cli/VPhoneCameraServer.swift @@ -0,0 +1,255 @@ +import CoreGraphics +import Foundation +import Virtualization + +/// Host-side virtual-camera server. +/// +/// Opens a vsock connection to the guest on port 1338 (separate from the +/// vphoned control channel on 1337) and pushes raw BGRA frames at a fixed +/// rate. The guest counterpart (a libvcamcaptured-attached receiver inside +/// cameracaptured) ferries those frames into the AVF capture pipeline. +/// +/// Wire format (one frame, length-prefixed): +/// uint32 LE total_payload_length +/// uint32 LE header_json_length +/// bytes header JSON (UTF-8), keys: w, h, bpr, fmt, ts +/// bytes raw pixel data, exactly `bpr * h` bytes +/// +/// The header carries the format so the receiver doesn't have to +/// assume; for now the producer fixes width/height/bpr/fmt at +/// 1280x720 BGRA. Future producers may emit different formats. +@MainActor +final class VPhoneCameraServer { + enum SourceKind: String { + case off + case testPattern + case videoFile // .mov / .mp4 / .m4v via AVAssetReader + } + + nonisolated static let vsockPort: UInt32 = 1338 + nonisolated static let defaultWidth: Int = 1280 + nonisolated static let defaultHeight: Int = 720 + nonisolated static let defaultFPS: Double = 30.0 + nonisolated static let pixelFormat: UInt32 = 0x4247_5241 // 'BGRA' — kCMPixelFormat_32BGRA + + private(set) var sourceKind: SourceKind = .off + private(set) var isConnected = false + + private var device: VZVirtioSocketDevice? + private var connection: VZVirtioSocketConnection? + private var connectionFD: Int32 = -1 + private var producer: VPhoneFrameProducer? + private var timer: DispatchSourceTimer? + private var connectionAttemptToken: UInt64 = 0 + + private let sendQueue = DispatchQueue( + label: "com.vphone.camera.send", qos: .userInteractive) + private let producerQueue = DispatchQueue( + label: "com.vphone.camera.producer", qos: .userInteractive) + + var onConnectionStateChange: ((Bool) -> Void)? + + // MARK: - Lifecycle + + func connect(device: VZVirtioSocketDevice) { + self.device = device + attemptConnect() + } + + func disconnect() { + stopStreaming() + if connectionFD >= 0 { + close(connectionFD) + connectionFD = -1 + } + connection = nil + if isConnected { + isConnected = false + onConnectionStateChange?(false) + } + } + + // MARK: - Source selection + + func setSource(_ kind: SourceKind, videoURL: URL? = nil) { + if sourceKind == kind, kind != .videoFile { return } + let wasStreaming = (timer != nil) + if wasStreaming { stopStreaming() } + sourceKind = kind + switch kind { + case .off: + producer = nil + case .testPattern: + producer = VPhoneTestPatternProducer( + width: Self.defaultWidth, + height: Self.defaultHeight) + case .videoFile: + guard let url = videoURL else { + print("[camera] videoFile source requires a URL") + producer = nil + sourceKind = .off + return + } + do { + producer = try VPhoneVideoFileProducer( + url: url, + width: Self.defaultWidth, + height: Self.defaultHeight) + print("[camera] video file source = \(url.lastPathComponent)") + } catch { + print("[camera] failed to open \(url.lastPathComponent): \(error)") + producer = nil + sourceKind = .off + } + } + if wasStreaming, producer != nil { + startStreaming() + } + } + + // MARK: - Streaming + + func startStreaming() { + guard producer != nil, isConnected else { return } + if timer != nil { return } + let interval = 1.0 / Self.defaultFPS + // Timer fires on the main queue so MainActor-isolated state + // (producer, connectionFD) can be read directly without tripping + // Swift 6's strict-concurrency isolation check. The frame + // production + send is then hopped onto producerQueue so + // BGRA generation doesn't run on the UI thread. + let t = DispatchSource.makeTimerSource(queue: DispatchQueue.main) + t.schedule(deadline: .now(), repeating: interval, leeway: .milliseconds(2)) + t.setEventHandler { [weak self] in + guard let self else { return } + guard let producer = self.producer else { return } + let fd = self.connectionFD + guard fd >= 0 else { return } + let q = self.producerQueue + q.async { + guard let frame = producer.nextFrame() else { return } + let ok = Self.send(fd: fd, frame: frame) + if !ok { + Task { @MainActor [weak self] in + guard let self else { return } + // Avoid double-handling if a parallel write already + // dropped the connection. + if self.connectionFD == fd { + self.handleDisconnect() + } + } + } + } + } + timer = t + t.resume() + print("[camera] streaming started — source=\(sourceKind.rawValue)") + } + + func stopStreaming() { + guard let t = timer else { return } + t.cancel() + timer = nil + print("[camera] streaming stopped") + } + + /// Mark the current connection dead and start reconnecting. Called when + /// a write fails with EPIPE — the most common case is vphoned auto-update + /// killing its server-side socket while the host is streaming. + private func handleDisconnect() { + print("[camera] disconnect detected, will reconnect") + let oldFD = connectionFD + connectionFD = -1 + connection = nil + if isConnected { + isConnected = false + onConnectionStateChange?(false) + } + if oldFD >= 0 { close(oldFD) } + // Note: streaming timer continues running but ticks no-op until + // connectionFD becomes valid again. + attemptConnect() + } + + // MARK: - Connect + + private func attemptConnect() { + guard let device else { return } + connectionAttemptToken &+= 1 + let attemptToken = connectionAttemptToken + device.connect(toPort: Self.vsockPort) { + [weak self] (result: Result) in + Task { @MainActor in + guard let self else { return } + guard self.connectionAttemptToken == attemptToken else { return } + switch result { + case let .success(conn): + self.connection = conn + self.connectionFD = conn.fileDescriptor + self.isConnected = true + print("[camera] connected on vsock port \(Self.vsockPort)") + self.onConnectionStateChange?(true) + if self.sourceKind != .off { self.startStreaming() } + case let .failure(error): + print("[camera] connect failed: \(error). Retrying in 3s.") + DispatchQueue.main.asyncAfter(deadline: .now() + 3.0) { [weak self] in + Task { @MainActor in + guard let self else { return } + guard self.connectionAttemptToken == attemptToken else { return } + self.attemptConnect() + } + } + } + } + } + } + + // MARK: - Wire + + @discardableResult + nonisolated private static func send(fd: Int32, frame: VPhoneCameraFrame) -> Bool { + // header + let headerDict: [String: Any] = [ + "w": frame.width, + "h": frame.height, + "bpr": frame.bytesPerRow, + "fmt": Self.pixelFormat, + "ts": frame.timestampNS, + ] + guard + let headerData = try? JSONSerialization.data(withJSONObject: headerDict, options: []) + else { return false } + let totalLen = UInt32(4 + headerData.count + frame.pixels.count) + let headerLen = UInt32(headerData.count) + var prefix = Data() + prefix.append(contentsOf: withUnsafeBytes(of: totalLen.littleEndian) { Array($0) }) + prefix.append(contentsOf: withUnsafeBytes(of: headerLen.littleEndian) { Array($0) }) + // Concat all into one buffer to make a single write — short frames + // are cheap; large ones (1280*720*4 ≈ 3.5 MB) still fit comfortably + // in a vsock send buffer and a single write keeps frame integrity + // even if a future reader uses non-buffered I/O. + var out = Data(capacity: Int(totalLen) + 4) + out.append(prefix) + out.append(headerData) + out.append(frame.pixels) + var ok = true + out.withUnsafeBytes { bytes -> Void in + guard let base = bytes.baseAddress else { ok = false; return } + var remaining = bytes.count + var cursor = base + while remaining > 0 { + let n = write(fd, cursor, remaining) + if n < 0 { + if errno == EINTR { continue } + print("[camera] write errno=\(errno)") + ok = false + return + } + if n == 0 { ok = false; return } + remaining -= n + cursor = cursor.advanced(by: n) + } + } + return ok + } +} diff --git a/sources/vphone-cli/VPhoneFrameProducer.swift b/sources/vphone-cli/VPhoneFrameProducer.swift new file mode 100644 index 0000000..cb587fd --- /dev/null +++ b/sources/vphone-cli/VPhoneFrameProducer.swift @@ -0,0 +1,249 @@ +import AVFoundation +import CoreGraphics +import CoreImage +import CoreVideo +import Foundation + +/// A single BGRA frame produced by a frame source. +struct VPhoneCameraFrame: Sendable { + let width: Int + let height: Int + let bytesPerRow: Int + let timestampNS: UInt64 + let pixels: Data +} + +/// Frame source for the host-side virtual camera server. +/// +/// `@unchecked Sendable` because each conforming producer is mutated only +/// from the camera server's producer queue (single writer); Swift can't +/// see the queue isolation, so we opt out of the strict-concurrency +/// check. +protocol VPhoneFrameProducer: AnyObject, Sendable { + func nextFrame() -> VPhoneCameraFrame? +} + +// MARK: - Test pattern + +/// Generates a smoothly-animating BGRA pattern: a moving vertical gradient +/// modulated by a sin wave, plus a frame counter overlay in the corner so +/// the receiver can verify that frames are advancing. No external assets +/// required. +final class VPhoneTestPatternProducer: VPhoneFrameProducer, @unchecked Sendable { + private let width: Int + private let height: Int + private let bytesPerRow: Int + private var frameIndex: UInt64 = 0 + private let startedAt: TimeInterval + + init(width: Int, height: Int) { + self.width = width + self.height = height + // Round bytesPerRow up to 16-byte alignment — common requirement for + // IOSurfaces and BGRA hardware paths. For 1280 width: 1280*4 = 5120 + // (already 16-aligned). + let stride = ((width * 4) + 15) & ~15 + self.bytesPerRow = stride + self.startedAt = ProcessInfo.processInfo.systemUptime + } + + func nextFrame() -> VPhoneCameraFrame? { + let now = ProcessInfo.processInfo.systemUptime + let elapsed = now - startedAt + let pixelCount = bytesPerRow * height + var bytes = [UInt8](repeating: 0, count: pixelCount) + + // Vertical gradient. Hue rolls with time. + let hueOffset = elapsed * 0.4 // turns per second + for y in 0.. (Double, Double, Double) { + let i = floor(h * 6.0) + let f = h * 6.0 - i + let p = v * (1.0 - s) + let q = v * (1.0 - f * s) + let t = v * (1.0 - (1.0 - f) * s) + switch Int(i) % 6 { + case 0: return (v, t, p) + case 1: return (q, v, p) + case 2: return (p, v, t) + case 3: return (p, q, v) + case 4: return (t, p, v) + default: return (v, p, q) + } + } +} + +// MARK: - Video file (.mov / .mp4 / .m4v via AVAssetReader) + +/// Plays a video file in a loop. Decode is delegated to `AVAssetReader` +/// with a BGRA output spec, so anything AVFoundation can demux on macOS +/// works (`.mov`, `.mp4`, `.m4v`). For unsupported containers +/// (`.mkv`, `.webm`, `.avi`) convert externally first +/// (e.g. `ffmpeg -i in.mkv -c copy out.mov` if codecs are compatible). +/// +/// The producer rescales the input video to the configured camera +/// width/height using a Core Image render so the wire-format payload +/// length stays constant regardless of the source resolution. Output is +/// always 8-bit BGRA, top-down, 16-byte aligned bytesPerRow. +final class VPhoneVideoFileProducer: VPhoneFrameProducer, @unchecked Sendable { + private let url: URL + private let width: Int + private let height: Int + private let bytesPerRow: Int + private var asset: AVURLAsset + private var reader: AVAssetReader? + private var readerOutput: AVAssetReaderTrackOutput? + private let ciContext: CIContext + + init(url: URL, width: Int, height: Int) throws { + self.url = url + self.width = width + self.height = height + self.bytesPerRow = ((width * 4) + 15) & ~15 + self.asset = AVURLAsset(url: url) + self.ciContext = CIContext(options: [.useSoftwareRenderer: false]) + try restartReader() + } + + private func restartReader() throws { + guard let track = asset.tracks(withMediaType: .video).first else { + throw NSError( + domain: "VPhoneVideoFileProducer", code: 1, + userInfo: [NSLocalizedDescriptionKey: + "\(url.lastPathComponent): no video track"]) + } + let r = try AVAssetReader(asset: asset) + let output = AVAssetReaderTrackOutput( + track: track, + outputSettings: [ + kCVPixelBufferPixelFormatTypeKey as String: + Int(kCVPixelFormatType_32BGRA), + ]) + output.alwaysCopiesSampleData = false + r.add(output) + guard r.startReading() else { + throw NSError( + domain: "VPhoneVideoFileProducer", code: 2, + userInfo: [NSLocalizedDescriptionKey: + "AVAssetReader.startReading failed: \(r.error?.localizedDescription ?? "?")"]) + } + self.reader = r + self.readerOutput = output + } + + func nextFrame() -> VPhoneCameraFrame? { + if reader?.status != .reading { + // Loop on EOF (or after an error) + do { try restartReader() } catch { print("[camera] mov restart failed: \(error)"); return nil } + } + guard let sb = readerOutput?.copyNextSampleBuffer(), + let pb = CMSampleBufferGetImageBuffer(sb) + else { + // EOF — restart on next call. + do { try restartReader() } catch {} + return nil + } + + let srcWidth = CVPixelBufferGetWidth(pb) + let srcHeight = CVPixelBufferGetHeight(pb) + + // Fast path: source already matches our requested dimensions and is + // BGRA — copy the planar bytes directly without going through Core + // Image. Saves the GPU render. + if srcWidth == width, srcHeight == height, + CVPixelBufferGetPixelFormatType(pb) == kCVPixelFormatType_32BGRA { + CVPixelBufferLockBaseAddress(pb, .readOnly) + defer { CVPixelBufferUnlockBaseAddress(pb, .readOnly) } + let srcBPR = CVPixelBufferGetBytesPerRow(pb) + guard let base = CVPixelBufferGetBaseAddress(pb) else { return nil } + var out = Data(count: bytesPerRow * height) + out.withUnsafeMutableBytes { dst in + let dstBase = dst.baseAddress! + for y in 0.. NSMenuItem { + let item = NSMenuItem(title: "Camera", action: nil, keyEquivalent: "") + let menu = NSMenu(title: "Camera") + + let status = NSMenuItem(title: "Camera server: disconnected", + action: nil, keyEquivalent: "") + status.isEnabled = false + cameraStatusItem = status + menu.addItem(status) + + menu.addItem(NSMenuItem.separator()) + + let off = makeItem("Source: Off", action: #selector(setCameraSourceOff)) + off.state = .on + cameraSourceOffItem = off + menu.addItem(off) + + let testPattern = makeItem("Source: Test Pattern", + action: #selector(setCameraSourceTestPattern)) + cameraSourceTestPatternItem = testPattern + menu.addItem(testPattern) + + let videoFile = makeItem("Source: Video File…", + action: #selector(setCameraSourceVideoFile)) + cameraSourceVideoFileItem = videoFile + menu.addItem(videoFile) + + menu.addItem(NSMenuItem.separator()) + + let startStop = makeItem("Start Streaming", + action: #selector(toggleCameraStreaming)) + startStop.isEnabled = false + cameraStartStopItem = startStop + menu.addItem(startStop) + + item.submenu = menu + return item + } + + func updateCameraConnectionState(connected: Bool) { + cameraStatusItem?.title = connected + ? "Camera server: connected (vsock 1338)" + : "Camera server: disconnected" + cameraStartStopItem?.isEnabled = connected && + (cameraServer?.sourceKind ?? .off) != .off + } + + private func refreshCameraSourceCheckmarks() { + let kind = cameraServer?.sourceKind ?? .off + cameraSourceOffItem?.state = (kind == .off) ? .on : .off + cameraSourceTestPatternItem?.state = + (kind == .testPattern) ? .on : .off + cameraSourceVideoFileItem?.state = + (kind == .videoFile) ? .on : .off + } + + @objc func setCameraSourceOff() { + cameraServer?.stopStreaming() + cameraServer?.setSource(.off) + refreshCameraSourceCheckmarks() + cameraStartStopItem?.isEnabled = false + cameraStartStopItem?.title = "Start Streaming" + } + + @objc func setCameraSourceTestPattern() { + cameraServer?.setSource(.testPattern) + refreshCameraSourceCheckmarks() + cameraStartStopItem?.isEnabled = (cameraServer?.isConnected ?? false) + } + + @objc func setCameraSourceVideoFile() { + let panel = NSOpenPanel() + panel.canChooseFiles = true + panel.canChooseDirectories = false + panel.allowsMultipleSelection = false + // AVFoundation's natively-decodable containers on macOS. Anything + // beyond these (.mkv/.webm/.avi) would need external conversion to + // .mov / .mp4 first. + panel.allowedContentTypes = [ + UTType(filenameExtension: "mov") ?? .movie, + UTType(filenameExtension: "mp4") ?? .movie, + UTType(filenameExtension: "m4v") ?? .movie, + ] + panel.prompt = "Use as Camera Source" + panel.title = "Pick a video file" + panel.runModal() + guard let url = panel.url else { return } + cameraServer?.setSource(.videoFile, videoURL: url) + refreshCameraSourceCheckmarks() + cameraStartStopItem?.isEnabled = + (cameraServer?.isConnected ?? false) && + (cameraServer?.sourceKind ?? .off) == .videoFile + } + + @objc func toggleCameraStreaming() { + guard let server = cameraServer else { return } + if cameraStartStopItem?.title == "Start Streaming" { + server.startStreaming() + cameraStartStopItem?.title = "Stop Streaming" + } else { + server.stopStreaming() + cameraStartStopItem?.title = "Start Streaming" + } + } +} diff --git a/sources/vphone-cli/VPhoneMenuConnect.swift b/sources/vphone-cli/VPhoneMenuConnect.swift index 78d22b6..3ff0088 100644 --- a/sources/vphone-cli/VPhoneMenuConnect.swift +++ b/sources/vphone-cli/VPhoneMenuConnect.swift @@ -65,6 +65,7 @@ extension VPhoneMenuController { menu.addItem(buildLocationSubmenu()) menu.addItem(buildBatterySubmenu()) + menu.addItem(buildCameraSubmenu()) item.submenu = menu return item diff --git a/sources/vphone-cli/VPhoneMenuController.swift b/sources/vphone-cli/VPhoneMenuController.swift index aa09e83..a30ffdc 100644 --- a/sources/vphone-cli/VPhoneMenuController.swift +++ b/sources/vphone-cli/VPhoneMenuController.swift @@ -35,6 +35,12 @@ class VPhoneMenuController { var locationReplayStopItem: NSMenuItem? var screenRecorder: VPhoneScreenRecorder? var recordingItem: NSMenuItem? + var cameraServer: VPhoneCameraServer? + var cameraStatusItem: NSMenuItem? + var cameraSourceOffItem: NSMenuItem? + var cameraSourceTestPatternItem: NSMenuItem? + var cameraSourceVideoFileItem: NSMenuItem? + var cameraStartStopItem: NSMenuItem? weak var captureView: VPhoneVirtualMachineView? var batterySyncEnabled = false var batterySyncStatusItem: NSMenuItem?