diff --git a/.github/failure-classes/FC-concurrent-capture-contention.json b/.github/failure-classes/FC-concurrent-capture-contention.json new file mode 100644 index 00000000000..e37985c7098 --- /dev/null +++ b/.github/failure-classes/FC-concurrent-capture-contention.json @@ -0,0 +1,9 @@ +{ + "schema_version": 1, + "id": "FC-concurrent-capture-contention", + "violated_contract": "Omi's periodic screen capture must yield while another app is actively capturing the screen (screenshot app, screen recorder, or call screen share); issuing ScreenCaptureKit requests during an external capture contends in WindowServer capture arbitration and can stall or terminate the other app's capture.", + "canonical_prevention": "Detect the external capture (frontmost screenshot app, share-indicator window) and pause the capture loop through ProactiveScreenshotCaptureGate with a post-capture backoff, instead of capturing and handling the fallout.", + "evidence_prs": [6819], + "scope_hints": ["desktop/macos/**"], + "status": "open" +} diff --git a/.github/scripts/product_file_line_count_ratchet_baseline/desktop-swift-proactiveassistants.json b/.github/scripts/product_file_line_count_ratchet_baseline/desktop-swift-proactiveassistants.json index 3b33b23d57b..08a1197f310 100644 --- a/.github/scripts/product_file_line_count_ratchet_baseline/desktop-swift-proactiveassistants.json +++ b/.github/scripts/product_file_line_count_ratchet_baseline/desktop-swift-proactiveassistants.json @@ -1,7 +1,7 @@ { "files": { "desktop/macos/Desktop/Sources/ProactiveAssistants/Assistants/TaskExtraction/TaskAssistant.swift": 1742, - "desktop/macos/Desktop/Sources/ProactiveAssistants/ProactiveAssistantsPlugin.swift": 1641 + "desktop/macos/Desktop/Sources/ProactiveAssistants/ProactiveAssistantsPlugin.swift": 1628 }, "raise_justifications": { "desktop/macos/Desktop/Sources/ProactiveAssistants/Assistants/TaskExtraction/TaskAssistant.swift": "Task candidate context now carries only backend action-item IDs, while local SQLite/staged evidence remains id-less so the model cannot target it for a backend mutation.", diff --git a/desktop/macos/Desktop/Sources/ConferencingApps.swift b/desktop/macos/Desktop/Sources/ConferencingApps.swift index 4465d3d539d..554b103b43a 100644 --- a/desktop/macos/Desktop/Sources/ConferencingApps.swift +++ b/desktop/macos/Desktop/Sources/ConferencingApps.swift @@ -146,6 +146,63 @@ enum ConferencingApps { return false } + // MARK: - Active outgoing screen share detection + + /// True if a single window — identified by owner app and title — is a share-indicator + /// window: the floating toolbar/status chrome a conferencing app shows **only while the + /// user is actively sharing their screen** in a call. + /// + /// Known signatures (window names are internal identifiers, not localized UI strings, + /// except the browser bubble which is English-locale best effort): + /// - Zoom: "zoom share statusbar window" / "zoom share toolbar window" floating controls + /// - Microsoft Teams: "Screen sharing toolbar" window while presenting + /// - Browsers (Google Meet / Teams web): the " is sharing your screen/a tab/a window" + /// stop-sharing bubble window + static func isShareIndicatorWindow(ownerName: String?, title: String?) -> Bool { + guard let ownerName = ownerName, let title = title, !title.isEmpty else { return false } + let lowerTitle = title.lowercased() + + if ownerName == "zoom.us" { + return lowerTitle.contains("zoom share") + } + + if ownerName.contains("Microsoft Teams") || ownerName == "MSTeams" { + return lowerTitle.contains("sharing toolbar") + } + + if browserApps.contains(ownerName) { + return lowerTitle.contains("is sharing your screen") + || lowerTitle.contains("is sharing a tab") + || lowerTitle.contains("is sharing a window") + } + + return false + } + + /// True if any on-screen window indicates an active outgoing screen share (the user is + /// presenting in Zoom/Teams/Meet/etc.). Window titles require Screen Recording permission; + /// without it only windows with readable names are considered. + /// + /// Used to pause Omi's periodic capture: a one-shot ScreenCaptureKit capture while another + /// app streams the screen contends in WindowServer capture arbitration and has been observed + /// to stop the other app's share (issue #10143). + static func activeScreenSharePresent() -> Bool { + guard + let windows = CGWindowListCopyWindowInfo( + [.optionOnScreenOnly, .excludeDesktopElements], kCGNullWindowID) as? [[String: Any]] + else { + return false + } + for window in windows { + let owner = window[kCGWindowOwnerName as String] as? String + let title = window[kCGWindowName as String] as? String + if isShareIndicatorWindow(ownerName: owner, title: title) { + return true + } + } + return false + } + // MARK: - CoreAudio process API (macOS 14.4+) — microphone-in-use detection @available(macOS 14.4, *) diff --git a/desktop/macos/Desktop/Sources/ProactiveAssistants/Core/ProactiveExternalCaptureYield.swift b/desktop/macos/Desktop/Sources/ProactiveAssistants/Core/ProactiveExternalCaptureYield.swift new file mode 100644 index 00000000000..fe60d3419d6 --- /dev/null +++ b/desktop/macos/Desktop/Sources/ProactiveAssistants/Core/ProactiveExternalCaptureYield.swift @@ -0,0 +1,84 @@ +import Foundation + +/// Decides, once per capture tick, whether Omi must skip its periodic screen capture to +/// yield to an **external capture** in progress: +/// +/// - a screenshot / screen-recording app frontmost (CleanShot, Shottr, macOS screenshot, …): +/// concurrent ScreenCaptureKit use stalls the user's capture UI for 20–60s (#6819), and +/// - an active outgoing call screen share (Zoom/Teams/Meet presenting): the same WindowServer +/// capture-arbitration contention has been observed to stop the user's share outright +/// (issue #10143). +/// +/// Both conditions run the same pause/backoff state machine (`ProactiveScreenshotCaptureGate`): +/// pause every tick while the condition holds, then hold a short backoff after it clears so the +/// other app's teardown/editor UI isn't disturbed either. +/// Failure class: FC-concurrent-capture-contention. +struct ProactiveExternalCaptureYield { + private(set) var screenshotGate = ProactiveScreenshotCaptureGate() + private(set) var shareGate = ProactiveScreenshotCaptureGate() + + /// True when this tick must skip capture. `isScreenShareActive` is an autoclosure so the + /// share check (a CGWindowList enumeration) is not evaluated while the screenshot gate + /// already pauses the tick. + mutating func shouldYield( + isScreenshotAppFrontmost: Bool, + isScreenShareActive: @autoclosure () -> Bool, + now: Date, + screenshotBackoffDuration: TimeInterval, + shareBackoffDuration: TimeInterval + ) -> Bool { + let wasScreenshotAppFrontmost = screenshotGate.wasScreenshotAppFrontmost + switch screenshotGate.nextDecision( + isScreenshotAppFrontmost: isScreenshotAppFrontmost, + now: now, + backoffDuration: screenshotBackoffDuration + ) { + case .pause: + if !wasScreenshotAppFrontmost { + log("ProactiveAssistantsPlugin: Screenshot app frontmost — pausing capture to avoid WindowServer contention") + } + return true + case .resumeIntoBackoff: + log( + "ProactiveAssistantsPlugin: Screenshot app no longer frontmost, holding backoff for \(Int(max(0, screenshotGate.backoffUntil.timeIntervalSinceNow)))s" + ) + return true + case .resumeAndCapture: + log("ProactiveAssistantsPlugin: Screenshot app no longer frontmost, holding backoff for 0s") + case .continueBackoff: + return true + case .capture: + break + } + + let wasScreenSharing = shareGate.wasScreenshotAppFrontmost + switch shareGate.nextDecision( + isScreenshotAppFrontmost: isScreenShareActive(), + now: now, + backoffDuration: shareBackoffDuration + ) { + case .pause: + if !wasScreenSharing { + log("ProactiveAssistantsPlugin: Active screen share detected — pausing capture until the share ends") + } + return true + case .resumeIntoBackoff: + log( + "ProactiveAssistantsPlugin: Screen share ended, holding backoff for \(Int(max(0, shareGate.backoffUntil.timeIntervalSinceNow)))s" + ) + return true + case .resumeAndCapture: + log("ProactiveAssistantsPlugin: Screen share ended, resuming capture") + return false + case .continueBackoff: + return true + case .capture: + return false + } + } + + mutating func reset() { + screenshotGate.reset() + shareGate.reset() + } +} diff --git a/desktop/macos/Desktop/Sources/ProactiveAssistants/ProactiveAssistantsPlugin.swift b/desktop/macos/Desktop/Sources/ProactiveAssistants/ProactiveAssistantsPlugin.swift index f0e791fb9be..d3e98d7b8d4 100644 --- a/desktop/macos/Desktop/Sources/ProactiveAssistants/ProactiveAssistantsPlugin.swift +++ b/desktop/macos/Desktop/Sources/ProactiveAssistants/ProactiveAssistantsPlugin.swift @@ -78,12 +78,14 @@ public class ProactiveAssistantsPlugin: NSObject { private var videoCallThrottleGate = ProactiveVideoCallThrottleGate() private let videoCallThrottleFactor = 5 // Capture 1 out of every 5 frames (effective ~5s interval) - // Screenshot-app yielding: pause capture entirely while another screenshot/recording - // app is frontmost, and hold a short backoff after it resigns so its editor UI isn't - // disturbed. Prevents Omi's 3s capture loop from locking WindowServer at the moment - // the user is trying to take a screenshot (CleanShot, Shottr, macOS screenshot, etc.). - private var screenshotCaptureGate = ProactiveScreenshotCaptureGate() + // External-capture yielding: pause capture entirely while a screenshot/recording app is + // frontmost (CleanShot, Shottr, macOS screenshot — WindowServer stalls, #6819) or while + // another app actively shares the screen in a call (Zoom/Teams/Meet presenting — the + // contention has been observed to stop the user's share, issue #10143). Each condition + // holds a short backoff after it clears. See ProactiveExternalCaptureYield. + private var externalCaptureYield = ProactiveExternalCaptureYield() private let screenshotAppBackoffDuration: TimeInterval = 10 + private let screenShareBackoffDuration: TimeInterval = 10 // Change-gated distribution: only distribute frames to assistants when context changes. // Eliminates continuous polling when the user stays on the same app/window. @@ -493,7 +495,7 @@ public class ProactiveAssistantsPlugin: NSObject { backgroundPollCount = 0 recoveryRetryCount = 0 isInDelayPeriod = false - screenshotCaptureGate.reset() + externalCaptureYield.reset() videoCallThrottleGate.reset() distributionGate.reset() latestCapturedFrame = nil @@ -694,31 +696,16 @@ public class ProactiveAssistantsPlugin: NSObject { return } - // Skip capture while a screenshot / screen-recording app is frontmost. - // Both apps using ScreenCaptureKit at the same time contend for WindowServer - // locks, which can stall the user's capture UI for 20-60s. Yield to the user. - let wasScreenshotAppFrontmostBeforeDecision = screenshotCaptureGate.wasScreenshotAppFrontmost - switch screenshotCaptureGate.nextDecision( + // Yield to an external capture in progress: a frontmost screenshot/recording app, or an + // active outgoing call screen share. See ProactiveExternalCaptureYield for rationale. + if externalCaptureYield.shouldYield( isScreenshotAppFrontmost: isScreenshotAppFrontmost(), + isScreenShareActive: ConferencingApps.activeScreenSharePresent(), now: now, - backoffDuration: screenshotAppBackoffDuration + screenshotBackoffDuration: screenshotAppBackoffDuration, + shareBackoffDuration: screenShareBackoffDuration ) { - case .pause: - if !wasScreenshotAppFrontmostBeforeDecision { - log("ProactiveAssistantsPlugin: Screenshot app frontmost — pausing capture to avoid WindowServer contention") - } - return - case .resumeIntoBackoff: - log( - "ProactiveAssistantsPlugin: Screenshot app no longer frontmost, holding backoff for \(Int(max(0, screenshotCaptureGate.backoffUntil.timeIntervalSinceNow)))s" - ) - return - case .resumeAndCapture: - log("ProactiveAssistantsPlugin: Screenshot app no longer frontmost, holding backoff for 0s") - case .continueBackoff: return - case .capture: - break } // Get current window info (use real app name, not cached) let (realAppName, windowTitle, windowID) = await WindowMonitor.getActiveWindowInfoAsync() diff --git a/desktop/macos/Desktop/Tests/AgentPillLifecycleTests.swift b/desktop/macos/Desktop/Tests/AgentPillLifecycleTests.swift index 8be56c44a98..a461fac1801 100644 --- a/desktop/macos/Desktop/Tests/AgentPillLifecycleTests.swift +++ b/desktop/macos/Desktop/Tests/AgentPillLifecycleTests.swift @@ -373,7 +373,9 @@ import XCTest XCTAssertFalse(windowSource.contains("resolveDelegationAndDispatch")) XCTAssertTrue(windowSource.contains("await dispatchChatQuery(")) XCTAssertFalse(source.contains("AgentPillFollowUpRoutingPolicy")) - XCTAssertTrue(source.contains("manager.continueAgent(from: pill, text: trimmed, attachments: staged)")) + // The bar has no typed pill composer since typing moved to the app's chat: + // pill steering routes to the main app instead of parsing wording locally. + XCTAssertTrue(source.contains("(NSApp.delegate as? AppDelegate)?.openMainAppWindow()")) } func testSubagentChatRendersMarkdownAndLargeBackHitTarget() throws { @@ -832,11 +834,14 @@ import XCTest XCTAssertFalse(body.contains("FloatingControlBarGeometry.targetFrame(")) } - func testSubagentComposerOnlyContinuesItsCanonicalSession() throws { + func testSubagentFollowUpsOnlyContinueTheCanonicalSession() throws { + // omi-test-quality: source-inspection -- static contract: the bar's typed composer is gone; follow-ups continue only through the manager bound to the pill's canonical session. let viewSource = try floatingControlBarViewSource() + let pillSource = try agentPillSource() XCTAssertFalse(viewSource.contains("AgentPillFollowUpRoutingPolicy")) - XCTAssertTrue(viewSource.contains("manager.continueAgent(from: pill, text: trimmed, attachments: staged)")) + XCTAssertTrue(pillSource.contains("guard pill.canonicalSessionId == sessionId else { return }")) + XCTAssertTrue(pillSource.contains("DesktopCoordinatorService.shared.continueAgent(")) } func testSpawnAgentToolCallOpensSubagentChat() throws { @@ -921,7 +926,10 @@ import XCTest let inputSource = String(viewSource[inputRange.lowerBound.. Bool { + yield.shouldYield( + isScreenshotAppFrontmost: screenshotApp, + isScreenShareActive: sharing, + now: start.addingTimeInterval(offset), + screenshotBackoffDuration: 10, + shareBackoffDuration: 10 + ) + } + + // No external capture: proceed. + XCTAssertFalse(tick(0, sharing: false)) + // Share starts: every tick yields for as long as the share lasts. + XCTAssertTrue(tick(1, sharing: true)) + XCTAssertTrue(tick(2, sharing: true)) + XCTAssertTrue(tick(120, sharing: true)) + // Share ends: still yield through the backoff window... + XCTAssertTrue(tick(121, sharing: false)) + XCTAssertTrue(tick(129, sharing: false)) + // ...and resume once the backoff has expired. + XCTAssertFalse(tick(131, sharing: false)) + } + + func testExternalCaptureYieldScreenshotGateShortCircuitsShareCheck() { + var yield = ProactiveExternalCaptureYield() + let now = Date(timeIntervalSinceReferenceDate: 9_500) + var shareChecked = false + + // While a screenshot app is frontmost, the (more expensive) share check must not run. + XCTAssertTrue( + yield.shouldYield( + isScreenshotAppFrontmost: true, + isScreenShareActive: { + shareChecked = true + return false + }(), + now: now, + screenshotBackoffDuration: 10, + shareBackoffDuration: 10 + ) + ) + XCTAssertFalse(shareChecked) + } + + func testExternalCaptureYieldResetClearsBothGates() { + var yield = ProactiveExternalCaptureYield() + let now = Date(timeIntervalSinceReferenceDate: 9_800) + + XCTAssertTrue( + yield.shouldYield( + isScreenshotAppFrontmost: true, + isScreenShareActive: true, + now: now, + screenshotBackoffDuration: 10, + shareBackoffDuration: 10 + ) + ) + yield.reset() + XCTAssertFalse(yield.screenshotGate.wasScreenshotAppFrontmost) + XCTAssertFalse(yield.shareGate.wasScreenshotAppFrontmost) + XCTAssertEqual(yield.screenshotGate.backoffUntil, .distantPast) + XCTAssertEqual(yield.shareGate.backoffUntil, .distantPast) + } + func testVideoCallThrottleGateCarriesCounterAndResetsWhenLeavingCall() { var gate = ProactiveVideoCallThrottleGate() diff --git a/desktop/macos/changelog/unreleased/20260721-pause-capture-during-screen-share.json b/desktop/macos/changelog/unreleased/20260721-pause-capture-during-screen-share.json new file mode 100644 index 00000000000..f22a6f47972 --- /dev/null +++ b/desktop/macos/changelog/unreleased/20260721-pause-capture-during-screen-share.json @@ -0,0 +1,3 @@ +{ + "change": "Fixed active Zoom/Teams/Meet screen shares stopping when Omi captured a screenshot — Omi now pauses its screen capture while you are presenting" +} diff --git a/desktop/macos/e2e/flows/screen-recording-permission.yaml b/desktop/macos/e2e/flows/screen-recording-permission.yaml index febefd40ec3..275ac9337a5 100644 --- a/desktop/macos/e2e/flows/screen-recording-permission.yaml +++ b/desktop/macos/e2e/flows/screen-recording-permission.yaml @@ -19,6 +19,7 @@ covers: - desktop/macos/Desktop/Sources/ProactiveAssistants/Assistants/MemoryExtraction/MemoryAssistant.swift - desktop/macos/Desktop/Sources/ProactiveAssistants/Core/ScreenCaptureTargetPolicy.swift - desktop/macos/Desktop/Sources/ProactiveAssistants/Core/ProactiveAssistantOrchestrationPolicy.swift + - desktop/macos/Desktop/Sources/ProactiveAssistants/Core/ProactiveExternalCaptureYield.swift - desktop/macos/Desktop/Sources/ProactiveAssistants/ProactiveAssistantsPlugin.swift - desktop/macos/Desktop/Sources/ProactiveAssistants/ProactiveAssistantsPlugin+NotificationSettings.swift - desktop/macos/Desktop/Sources/ProactiveAssistants/ProactiveAssistantsPlugin+ScreenCaptureHealth.swift