From 0223ad139c42341798fe658176b242440b6a9819 Mon Sep 17 00:00:00 2001 From: zqxwce Date: Mon, 15 Jun 2026 18:22:05 +0300 Subject: [PATCH] camera: Host-side virtual camera pipe (vphone-cli) Mac-side feeder for the virtual camera. The host generates frames (test pattern or video file) and pushes BGRA buffers over vsock to the guest daemon vphoned, which mirrors them into the shm region that the in-VM libvcamcaptured + libcamfix consume. VPhoneCameraServer @MainActor vsock client to vphoned port 1338. Reconnects on transport failure so the host can survive guest reboots. VPhoneFrameProducer Test-pattern (hue-rolling gradient) and VideoFile producers; emit BGRA at the session's requested width/height/fps. VPhoneMenuCamera Camera menu: pick source, start/stop the feeder, show stream status. VPhoneMenuController/Connect/AppDelegate Wire the menu into the existing menu bar and start the server alongside the other vsock services at VM boot. End state: with a vphoned listener on the guest side, host-generated BGRA frames appear in the shm region the camera daemon reads from. Co-Authored-By: Claude Opus 4.7 --- sources/vphone-cli/VPhoneAppDelegate.swift | 13 + sources/vphone-cli/VPhoneCameraServer.swift | 255 ++++++++++++++++++ sources/vphone-cli/VPhoneFrameProducer.swift | 249 +++++++++++++++++ sources/vphone-cli/VPhoneMenuCamera.swift | 111 ++++++++ sources/vphone-cli/VPhoneMenuConnect.swift | 1 + sources/vphone-cli/VPhoneMenuController.swift | 6 + 6 files changed, 635 insertions(+) create mode 100644 sources/vphone-cli/VPhoneCameraServer.swift create mode 100644 sources/vphone-cli/VPhoneFrameProducer.swift create mode 100644 sources/vphone-cli/VPhoneMenuCamera.swift diff --git a/sources/vphone-cli/VPhoneAppDelegate.swift b/sources/vphone-cli/VPhoneAppDelegate.swift index 34f6549..5a1ac34 100644 --- a/sources/vphone-cli/VPhoneAppDelegate.swift +++ b/sources/vphone-cli/VPhoneAppDelegate.swift @@ -13,6 +13,7 @@ class VPhoneAppDelegate: NSObject, NSApplicationDelegate { private var appWindowController: VPhoneAppWindowController? private var locationProvider: VPhoneLocationProvider? private var hostControl: VPhoneHostControl? + private var cameraServer: VPhoneCameraServer? private var sigintSource: DispatchSourceSignal? private var didAttemptAutoInstall = false @@ -88,8 +89,12 @@ class VPhoneAppDelegate: NSObject, NSApplicationDelegate { let provider = VPhoneLocationProvider(control: control) locationProvider = provider + let camServer = VPhoneCameraServer() + cameraServer = camServer + if let device = vm.virtualMachine.socketDevices.first as? VZVirtioSocketDevice { control.connect(device: device) + camServer.connect(device: device) } } @@ -135,6 +140,14 @@ class VPhoneAppDelegate: NSObject, NSApplicationDelegate { if let provider = locationProvider { mc.locationProvider = provider } + if let camServer = cameraServer { + mc.cameraServer = camServer + camServer.onConnectionStateChange = { [weak mc] connected in + Task { @MainActor in + mc?.updateCameraConnectionState(connected: connected) + } + } + } let recorder = VPhoneScreenRecorder() mc.screenRecorder = recorder menuController = mc diff --git a/sources/vphone-cli/VPhoneCameraServer.swift b/sources/vphone-cli/VPhoneCameraServer.swift new file mode 100644 index 0000000..d84e4f2 --- /dev/null +++ b/sources/vphone-cli/VPhoneCameraServer.swift @@ -0,0 +1,255 @@ +import CoreGraphics +import Foundation +import Virtualization + +/// Host-side virtual-camera server. +/// +/// Opens a vsock connection to the guest on port 1338 (separate from the +/// vphoned control channel on 1337) and pushes raw BGRA frames at a fixed +/// rate. The guest counterpart (a libvcamcaptured-attached receiver inside +/// cameracaptured) ferries those frames into the AVF capture pipeline. +/// +/// Wire format (one frame, length-prefixed): +/// uint32 LE total_payload_length +/// uint32 LE header_json_length +/// bytes header JSON (UTF-8), keys: w, h, bpr, fmt, ts +/// bytes raw pixel data, exactly `bpr * h` bytes +/// +/// The header carries the format so the receiver doesn't have to +/// assume; for now the producer fixes width/height/bpr/fmt at +/// 1280x720 BGRA. Future producers may emit different formats. +@MainActor +final class VPhoneCameraServer { + enum SourceKind: String { + case off + case testPattern + case videoFile // .mov / .mp4 / .m4v via AVAssetReader + } + + nonisolated static let vsockPort: UInt32 = 1338 + nonisolated static let defaultWidth: Int = 1280 + nonisolated static let defaultHeight: Int = 720 + nonisolated static let defaultFPS: Double = 30.0 + nonisolated static let pixelFormat: UInt32 = 0x4247_5241 // 'BGRA' — kCMPixelFormat_32BGRA + + private(set) var sourceKind: SourceKind = .off + private(set) var isConnected = false + + private var device: VZVirtioSocketDevice? + private var connection: VZVirtioSocketConnection? + private var connectionFD: Int32 = -1 + private var producer: VPhoneFrameProducer? + private var timer: DispatchSourceTimer? + private var connectionAttemptToken: UInt64 = 0 + + private let sendQueue = DispatchQueue( + label: "com.vphone.camera.send", qos: .userInteractive) + private let producerQueue = DispatchQueue( + label: "com.vphone.camera.producer", qos: .userInteractive) + + var onConnectionStateChange: ((Bool) -> Void)? + + // MARK: - Lifecycle + + func connect(device: VZVirtioSocketDevice) { + self.device = device + attemptConnect() + } + + func disconnect() { + stopStreaming() + if connectionFD >= 0 { + close(connectionFD) + connectionFD = -1 + } + connection = nil + if isConnected { + isConnected = false + onConnectionStateChange?(false) + } + } + + // MARK: - Source selection + + func setSource(_ kind: SourceKind, videoURL: URL? = nil) { + if sourceKind == kind, kind != .videoFile { return } + let wasStreaming = (timer != nil) + if wasStreaming { stopStreaming() } + sourceKind = kind + switch kind { + case .off: + producer = nil + case .testPattern: + producer = VPhoneTestPatternProducer( + width: Self.defaultWidth, + height: Self.defaultHeight) + case .videoFile: + guard let url = videoURL else { + print("[camera] videoFile source requires a URL") + producer = nil + sourceKind = .off + return + } + do { + producer = try VPhoneVideoFileProducer( + url: url, + width: Self.defaultWidth, + height: Self.defaultHeight) + print("[camera] video file source = \(url.lastPathComponent)") + } catch { + print("[camera] failed to open \(url.lastPathComponent): \(error)") + producer = nil + sourceKind = .off + } + } + if wasStreaming, producer != nil { + startStreaming() + } + } + + // MARK: - Streaming + + func startStreaming() { + guard producer != nil, isConnected else { return } + if timer != nil { return } + let interval = 1.0 / Self.defaultFPS + // Timer fires on the main queue so MainActor-isolated state + // (producer, connectionFD) can be read directly without tripping + // Swift 6's strict-concurrency isolation check. The frame + // production + send is then hopped onto producerQueue so + // BGRA generation doesn't run on the UI thread. + let t = DispatchSource.makeTimerSource(queue: DispatchQueue.main) + t.schedule(deadline: .now(), repeating: interval, leeway: .milliseconds(2)) + t.setEventHandler { [weak self] in + guard let self else { return } + guard let producer = self.producer else { return } + let fd = self.connectionFD + guard fd >= 0 else { return } + let q = self.producerQueue + q.async { + guard let frame = producer.nextFrame() else { return } + let ok = Self.send(fd: fd, frame: frame) + if !ok { + Task { @MainActor [weak self] in + guard let self else { return } + // Avoid double-handling if a parallel write already + // dropped the connection. + if self.connectionFD == fd { + self.handleDisconnect() + } + } + } + } + } + timer = t + t.resume() + print("[camera] streaming started — source=\(sourceKind.rawValue)") + } + + func stopStreaming() { + guard let t = timer else { return } + t.cancel() + timer = nil + print("[camera] streaming stopped") + } + + /// Mark the current connection dead and start reconnecting. Called when + /// a write fails with EPIPE — the most common case is vphoned auto-update + /// killing its server-side socket while the host is streaming. + private func handleDisconnect() { + print("[camera] disconnect detected, will reconnect") + let oldFD = connectionFD + connectionFD = -1 + connection = nil + if isConnected { + isConnected = false + onConnectionStateChange?(false) + } + if oldFD >= 0 { close(oldFD) } + // Note: streaming timer continues running but ticks no-op until + // connectionFD becomes valid again. + attemptConnect() + } + + // MARK: - Connect + + private func attemptConnect() { + guard let device else { return } + connectionAttemptToken &+= 1 + let attemptToken = connectionAttemptToken + device.connect(toPort: Self.vsockPort) { + [weak self] (result: Result) in + Task { @MainActor in + guard let self else { return } + guard self.connectionAttemptToken == attemptToken else { return } + switch result { + case let .success(conn): + self.connection = conn + self.connectionFD = conn.fileDescriptor + self.isConnected = true + print("[camera] connected on vsock port \(Self.vsockPort)") + self.onConnectionStateChange?(true) + if self.sourceKind != .off { self.startStreaming() } + case let .failure(error): + print("[camera] connect failed: \(error). Retrying in 3s.") + DispatchQueue.main.asyncAfter(deadline: .now() + 3.0) { [weak self] in + Task { @MainActor in + guard let self else { return } + guard self.connectionAttemptToken == attemptToken else { return } + self.attemptConnect() + } + } + } + } + } + } + + // MARK: - Wire + + @discardableResult + nonisolated private static func send(fd: Int32, frame: VPhoneCameraFrame) -> Bool { + // header + let headerDict: [String: Any] = [ + "w": frame.width, + "h": frame.height, + "bpr": frame.bytesPerRow, + "fmt": Self.pixelFormat, + "ts": frame.timestampNS, + ] + guard + let headerData = try? JSONSerialization.data(withJSONObject: headerDict, options: []) + else { return false } + let totalLen = UInt32(4 + headerData.count + frame.pixels.count) + let headerLen = UInt32(headerData.count) + var prefix = Data() + prefix.append(contentsOf: withUnsafeBytes(of: totalLen.littleEndian) { Array($0) }) + prefix.append(contentsOf: withUnsafeBytes(of: headerLen.littleEndian) { Array($0) }) + // Concat all into one buffer to make a single write — short frames + // are cheap; large ones (1280*720*4 ≈ 3.5 MB) still fit comfortably + // in a vsock send buffer and a single write keeps frame integrity + // even if a future reader uses non-buffered I/O. + var out = Data(capacity: Int(totalLen) + 4) + out.append(prefix) + out.append(headerData) + out.append(frame.pixels) + var ok = true + out.withUnsafeBytes { bytes -> Void in + guard let base = bytes.baseAddress else { ok = false; return } + var remaining = bytes.count + var cursor = base + while remaining > 0 { + let n = write(fd, cursor, remaining) + if n < 0 { + if errno == EINTR { continue } + print("[camera] write errno=\(errno)") + ok = false + return + } + if n == 0 { ok = false; return } + remaining -= n + cursor = cursor.advanced(by: n) + } + } + return ok + } +} diff --git a/sources/vphone-cli/VPhoneFrameProducer.swift b/sources/vphone-cli/VPhoneFrameProducer.swift new file mode 100644 index 0000000..cb587fd --- /dev/null +++ b/sources/vphone-cli/VPhoneFrameProducer.swift @@ -0,0 +1,249 @@ +import AVFoundation +import CoreGraphics +import CoreImage +import CoreVideo +import Foundation + +/// A single BGRA frame produced by a frame source. +struct VPhoneCameraFrame: Sendable { + let width: Int + let height: Int + let bytesPerRow: Int + let timestampNS: UInt64 + let pixels: Data +} + +/// Frame source for the host-side virtual camera server. +/// +/// `@unchecked Sendable` because each conforming producer is mutated only +/// from the camera server's producer queue (single writer); Swift can't +/// see the queue isolation, so we opt out of the strict-concurrency +/// check. +protocol VPhoneFrameProducer: AnyObject, Sendable { + func nextFrame() -> VPhoneCameraFrame? +} + +// MARK: - Test pattern + +/// Generates a smoothly-animating BGRA pattern: a moving vertical gradient +/// modulated by a sin wave, plus a frame counter overlay in the corner so +/// the receiver can verify that frames are advancing. No external assets +/// required. +final class VPhoneTestPatternProducer: VPhoneFrameProducer, @unchecked Sendable { + private let width: Int + private let height: Int + private let bytesPerRow: Int + private var frameIndex: UInt64 = 0 + private let startedAt: TimeInterval + + init(width: Int, height: Int) { + self.width = width + self.height = height + // Round bytesPerRow up to 16-byte alignment — common requirement for + // IOSurfaces and BGRA hardware paths. For 1280 width: 1280*4 = 5120 + // (already 16-aligned). + let stride = ((width * 4) + 15) & ~15 + self.bytesPerRow = stride + self.startedAt = ProcessInfo.processInfo.systemUptime + } + + func nextFrame() -> VPhoneCameraFrame? { + let now = ProcessInfo.processInfo.systemUptime + let elapsed = now - startedAt + let pixelCount = bytesPerRow * height + var bytes = [UInt8](repeating: 0, count: pixelCount) + + // Vertical gradient. Hue rolls with time. + let hueOffset = elapsed * 0.4 // turns per second + for y in 0.. (Double, Double, Double) { + let i = floor(h * 6.0) + let f = h * 6.0 - i + let p = v * (1.0 - s) + let q = v * (1.0 - f * s) + let t = v * (1.0 - (1.0 - f) * s) + switch Int(i) % 6 { + case 0: return (v, t, p) + case 1: return (q, v, p) + case 2: return (p, v, t) + case 3: return (p, q, v) + case 4: return (t, p, v) + default: return (v, p, q) + } + } +} + +// MARK: - Video file (.mov / .mp4 / .m4v via AVAssetReader) + +/// Plays a video file in a loop. Decode is delegated to `AVAssetReader` +/// with a BGRA output spec, so anything AVFoundation can demux on macOS +/// works (`.mov`, `.mp4`, `.m4v`). For unsupported containers +/// (`.mkv`, `.webm`, `.avi`) convert externally first +/// (e.g. `ffmpeg -i in.mkv -c copy out.mov` if codecs are compatible). +/// +/// The producer rescales the input video to the configured camera +/// width/height using a Core Image render so the wire-format payload +/// length stays constant regardless of the source resolution. Output is +/// always 8-bit BGRA, top-down, 16-byte aligned bytesPerRow. +final class VPhoneVideoFileProducer: VPhoneFrameProducer, @unchecked Sendable { + private let url: URL + private let width: Int + private let height: Int + private let bytesPerRow: Int + private var asset: AVURLAsset + private var reader: AVAssetReader? + private var readerOutput: AVAssetReaderTrackOutput? + private let ciContext: CIContext + + init(url: URL, width: Int, height: Int) throws { + self.url = url + self.width = width + self.height = height + self.bytesPerRow = ((width * 4) + 15) & ~15 + self.asset = AVURLAsset(url: url) + self.ciContext = CIContext(options: [.useSoftwareRenderer: false]) + try restartReader() + } + + private func restartReader() throws { + guard let track = asset.tracks(withMediaType: .video).first else { + throw NSError( + domain: "VPhoneVideoFileProducer", code: 1, + userInfo: [NSLocalizedDescriptionKey: + "\(url.lastPathComponent): no video track"]) + } + let r = try AVAssetReader(asset: asset) + let output = AVAssetReaderTrackOutput( + track: track, + outputSettings: [ + kCVPixelBufferPixelFormatTypeKey as String: + Int(kCVPixelFormatType_32BGRA), + ]) + output.alwaysCopiesSampleData = false + r.add(output) + guard r.startReading() else { + throw NSError( + domain: "VPhoneVideoFileProducer", code: 2, + userInfo: [NSLocalizedDescriptionKey: + "AVAssetReader.startReading failed: \(r.error?.localizedDescription ?? "?")"]) + } + self.reader = r + self.readerOutput = output + } + + func nextFrame() -> VPhoneCameraFrame? { + if reader?.status != .reading { + // Loop on EOF (or after an error) + do { try restartReader() } catch { print("[camera] mov restart failed: \(error)"); return nil } + } + guard let sb = readerOutput?.copyNextSampleBuffer(), + let pb = CMSampleBufferGetImageBuffer(sb) + else { + // EOF — restart on next call. + do { try restartReader() } catch {} + return nil + } + + let srcWidth = CVPixelBufferGetWidth(pb) + let srcHeight = CVPixelBufferGetHeight(pb) + + // Fast path: source already matches our requested dimensions and is + // BGRA — copy the planar bytes directly without going through Core + // Image. Saves the GPU render. + if srcWidth == width, srcHeight == height, + CVPixelBufferGetPixelFormatType(pb) == kCVPixelFormatType_32BGRA { + CVPixelBufferLockBaseAddress(pb, .readOnly) + defer { CVPixelBufferUnlockBaseAddress(pb, .readOnly) } + let srcBPR = CVPixelBufferGetBytesPerRow(pb) + guard let base = CVPixelBufferGetBaseAddress(pb) else { return nil } + var out = Data(count: bytesPerRow * height) + out.withUnsafeMutableBytes { dst in + let dstBase = dst.baseAddress! + for y in 0.. NSMenuItem { + let item = NSMenuItem(title: "Camera", action: nil, keyEquivalent: "") + let menu = NSMenu(title: "Camera") + + let status = NSMenuItem(title: "Camera server: disconnected", + action: nil, keyEquivalent: "") + status.isEnabled = false + cameraStatusItem = status + menu.addItem(status) + + menu.addItem(NSMenuItem.separator()) + + let off = makeItem("Source: Off", action: #selector(setCameraSourceOff)) + off.state = .on + cameraSourceOffItem = off + menu.addItem(off) + + let testPattern = makeItem("Source: Test Pattern", + action: #selector(setCameraSourceTestPattern)) + cameraSourceTestPatternItem = testPattern + menu.addItem(testPattern) + + let videoFile = makeItem("Source: Video File…", + action: #selector(setCameraSourceVideoFile)) + cameraSourceVideoFileItem = videoFile + menu.addItem(videoFile) + + menu.addItem(NSMenuItem.separator()) + + let startStop = makeItem("Start Streaming", + action: #selector(toggleCameraStreaming)) + startStop.isEnabled = false + cameraStartStopItem = startStop + menu.addItem(startStop) + + item.submenu = menu + return item + } + + func updateCameraConnectionState(connected: Bool) { + cameraStatusItem?.title = connected + ? "Camera server: connected (vsock 1338)" + : "Camera server: disconnected" + cameraStartStopItem?.isEnabled = connected && + (cameraServer?.sourceKind ?? .off) != .off + } + + private func refreshCameraSourceCheckmarks() { + let kind = cameraServer?.sourceKind ?? .off + cameraSourceOffItem?.state = (kind == .off) ? .on : .off + cameraSourceTestPatternItem?.state = + (kind == .testPattern) ? .on : .off + cameraSourceVideoFileItem?.state = + (kind == .videoFile) ? .on : .off + } + + @objc func setCameraSourceOff() { + cameraServer?.stopStreaming() + cameraServer?.setSource(.off) + refreshCameraSourceCheckmarks() + cameraStartStopItem?.isEnabled = false + cameraStartStopItem?.title = "Start Streaming" + } + + @objc func setCameraSourceTestPattern() { + cameraServer?.setSource(.testPattern) + refreshCameraSourceCheckmarks() + cameraStartStopItem?.isEnabled = (cameraServer?.isConnected ?? false) + } + + @objc func setCameraSourceVideoFile() { + let panel = NSOpenPanel() + panel.canChooseFiles = true + panel.canChooseDirectories = false + panel.allowsMultipleSelection = false + // AVFoundation's natively-decodable containers on macOS. Anything + // beyond these (.mkv/.webm/.avi) would need external conversion to + // .mov / .mp4 first. + panel.allowedContentTypes = [ + UTType(filenameExtension: "mov") ?? .movie, + UTType(filenameExtension: "mp4") ?? .movie, + UTType(filenameExtension: "m4v") ?? .movie, + ] + panel.prompt = "Use as Camera Source" + panel.title = "Pick a video file" + panel.runModal() + guard let url = panel.url else { return } + cameraServer?.setSource(.videoFile, videoURL: url) + refreshCameraSourceCheckmarks() + cameraStartStopItem?.isEnabled = + (cameraServer?.isConnected ?? false) && + (cameraServer?.sourceKind ?? .off) == .videoFile + } + + @objc func toggleCameraStreaming() { + guard let server = cameraServer else { return } + if cameraStartStopItem?.title == "Start Streaming" { + server.startStreaming() + cameraStartStopItem?.title = "Stop Streaming" + } else { + server.stopStreaming() + cameraStartStopItem?.title = "Start Streaming" + } + } +} diff --git a/sources/vphone-cli/VPhoneMenuConnect.swift b/sources/vphone-cli/VPhoneMenuConnect.swift index 78d22b6..3ff0088 100644 --- a/sources/vphone-cli/VPhoneMenuConnect.swift +++ b/sources/vphone-cli/VPhoneMenuConnect.swift @@ -65,6 +65,7 @@ extension VPhoneMenuController { menu.addItem(buildLocationSubmenu()) menu.addItem(buildBatterySubmenu()) + menu.addItem(buildCameraSubmenu()) item.submenu = menu return item diff --git a/sources/vphone-cli/VPhoneMenuController.swift b/sources/vphone-cli/VPhoneMenuController.swift index aa09e83..a30ffdc 100644 --- a/sources/vphone-cli/VPhoneMenuController.swift +++ b/sources/vphone-cli/VPhoneMenuController.swift @@ -35,6 +35,12 @@ class VPhoneMenuController { var locationReplayStopItem: NSMenuItem? var screenRecorder: VPhoneScreenRecorder? var recordingItem: NSMenuItem? + var cameraServer: VPhoneCameraServer? + var cameraStatusItem: NSMenuItem? + var cameraSourceOffItem: NSMenuItem? + var cameraSourceTestPatternItem: NSMenuItem? + var cameraSourceVideoFileItem: NSMenuItem? + var cameraStartStopItem: NSMenuItem? weak var captureView: VPhoneVirtualMachineView? var batterySyncEnabled = false var batterySyncStatusItem: NSMenuItem?