Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion Resources/analytics-events.psv
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@ app_launched|
app_session_stall_detected|app_version,build_channel,build_revision,build_version,duration_bucket,format_ready,heartbeat_age_bucket,last_event,os_major,previous_clean_shutdown,reason,recovering,session_active,session_duration_bucket,session_kind,session_stage,stall_kind,stall_stage,trigger
app_unclean_shutdown_detected|app_version,build_channel,build_revision,build_version,duration_bucket,format_ready,heartbeat_age_bucket,last_event,os_major,previous_clean_shutdown,reason,recovering,session_active,session_duration_bucket,session_kind,session_stage,stall_kind,stall_stage,trigger
dictation_artifact_saved|delivery,duration_bucket,save_outcome,surface,trigger,word_count_bucket
dictation_audio_route_changed|default_input_class,default_output_class,format_ready,hfp_suspected,input_channels,input_device_class,input_rate_hz,output_channels,output_device_class,output_rate_hz,recovering,recovery_latency_bucket,route_shape,sample_flow_started,selected_input_class,selection_overrode_default,selection_reason,was_recording
dictation_audio_route_changed|default_input_class,default_output_class,format_ready,hfp_suspected,input_device_class,output_device_class,recovering,route_shape,sample_flow_started,selected_input_class,selection_overrode_default,selection_reason,was_recording
dictation_audio_route_recovery_finished|default_input_class,default_output_class,format_ready,hfp_suspected,input_channels,input_device_class,input_rate_hz,outcome,output_channels,output_device_class,output_rate_hz,recovering,recovery_latency_bucket,route_shape,sample_flow_started,selected_input_class,selection_overrode_default,selection_reason,was_recording
dictation_audio_route_recovery_timeout|default_input_class,default_output_class,format_ready,hfp_suspected,input_channels,input_device_class,input_rate_hz,output_channels,output_device_class,output_rate_hz,recovering,recovery_latency_bucket,route_shape,sample_flow_started,selected_input_class,selection_overrode_default,selection_reason,was_recording
dictation_cancelled|default_input_class,default_output_class,duration_bucket,format_ready,hfp_suspected,input_channels,input_device_class,input_rate_hz,output_channels,output_device_class,output_rate_hz,recovering,recovery_latency_bucket,route_shape,sample_flow_started,selected_input_class,selection_overrode_default,selection_reason,trigger,was_recording
Expand All @@ -33,6 +33,7 @@ dictation_recording_too_short|default_input_class,default_output_class,duration_
dictation_start_failed|default_input_class,default_output_class,failure_kind,format_ready,hfp_suspected,input_channels,input_device_class,input_rate_hz,output_channels,output_device_class,output_rate_hz,recovering,recovery_latency_bucket,route_shape,sample_flow_started,selected_input_class,selection_overrode_default,selection_reason,start_attempt_bucket,trigger,was_recording
dictation_started|default_input_class,default_output_class,format_ready,hfp_suspected,input_channels,input_device_class,input_rate_hz,output_channels,output_device_class,output_rate_hz,recovering,recovery_latency_bucket,route_shape,sample_flow_started,selected_input_class,selection_overrode_default,selection_reason,trigger,was_recording
dictation_stop_latency_measured|auto_enter_bucket,auto_send,auto_send_block_reason,auto_send_expected,auto_send_key,cleanup_bucket,cleanup_changed,cleanup_enabled,copy_reason,decode_bucket,default_input_class,default_output_class,delivery,format_ready,hfp_suspected,input_channels,input_device_class,input_rate_hz,mic_stop_bucket,model_wait_bucket,outcome,output_channels,output_device_class,output_rate_hz,paste_bucket,recovering,recovery_latency_bucket,route_shape,sample_flow_started,save_bucket,save_outcome,selected_input_class,selection_overrode_default,selection_reason,stop_to_done_bucket,stop_to_paste_bucket,target_confirmation_mode,trigger,was_recording,word_count_bucket
dictation_zombie_recovery_finished|failure_kind,hfp_suspected,input_device_class,output_device_class,result,route_shape,stage
local_summary_cancelled|duration_bucket,provider,result,runtime,setup_ready,stage,summary_action
local_summary_failed|duration_bucket,failure_kind,provider,result,runtime,setup_ready,stage,summary_action
local_summary_finished|chunk_count_bucket,duration_bucket,provider,result,runtime,setup_ready,stage,summary_action
Expand Down
2 changes: 2 additions & 0 deletions Sources/Observability/CLAUDE.md
Original file line number Diff line number Diff line change
Expand Up @@ -52,6 +52,8 @@ anonymous analytics, and Sparkle update plumbing.
- PostHog config is read from `Info.plist` (`TranscriptedPostHogAPIKey`, `TranscriptedPostHogHost`) or process environment (`POSTHOG_API_KEY`, `POSTHOG_HOST`), and anonymous analytics must stay event-allowlisted and bucketed rather than sending raw payloads
- New analytics events should be added to `Resources/analytics-events.psv`; new reviewed non-bucket property names should be added to `Resources/analytics-reviewed-properties.psv`; run `python3 scripts/ops/normalize-analytics-taxonomy.py --check` after edits or union merges.
- Activation analytics should route through `ActivationTelemetry` so artifact action, agent prompt/setup, and saved-recent artifact return-proxy events keep stable names, targets, result enums, and coarse age/window buckets.
- Timeline analytics should route through `TimelineAnalyticsTelemetry` so future Dayflow events keep screen-derived data local and only send coarse enum/bucket payloads.
- `dictation_zombie_recovery_finished` is the single PostHog terminal event for each zombie-engine attempt. Keep its trigger, stage, result, and route fields categorical; raw device labels and exact sample counts are forbidden.
- Non-fatal error forwarding to Sentry is allowlisted. New `.error` events should not automatically assume they are safe to send off-device.
- `RuntimeDiagnostics` writes only coarse runtime state under app-owned state. Keep it free of transcript text, raw audio, file paths, device names, meeting titles, and speaker names.
- `ReliabilityPacketRecorder` derives packets from already-reviewed observability events (`EventReporter.capture` calls `ReliabilityPacketRecorder.record` directly — see the sink map in `docs/observability.md`). Keep its context allowlist coarse and bucketed; do not add raw error text, transcript text, raw audio, file paths, device names, meeting titles, speaker names, emails, tokens, or source app names.
Expand Down
2 changes: 2 additions & 0 deletions Sources/Observability/SentryEventPolicy.swift
Original file line number Diff line number Diff line change
Expand Up @@ -77,6 +77,7 @@ struct SentryEventPolicy: Equatable {
"readiness_refreshes",
"recovering",
"recovery_start_attempts",
"result",
"route_change_count_bucket",
"route_stability_warning",
"route_shape",
Expand All @@ -91,6 +92,7 @@ struct SentryEventPolicy: Equatable {
"session_stage",
"stall_kind",
"stall_stage",
"stage",
"start_attempts",
"stt_model",
"stop_timed_out",
Expand Down
2 changes: 2 additions & 0 deletions Sources/Speech/CLAUDE.md
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,8 @@
- `ParakeetEngine` mirrors `ParakeetRecoveryState` flags into `@Published var isRecovering` and `@Published var inputFormatReady` (forwarded through `STTRouter`). `DictationSessionController` uses `DictationReadinessWaitPolicy` in its wait-for-ready loop so it can distinguish between "still recovering", "refresh input readiness", and "safe to start" instead of blindly retrying. AirPods Hands-Free Profile (24kHz hw / 48kHz output bus) is supported: the tap is installed with `format: nil` so buffers arrive at the output rate, and `nativeSampleRate` tracks the output rate so downstream resampling is correct.
- `ParakeetEngine` consults `DictationInputDeviceSelectionPolicy` through `ParakeetAudioDeviceLookup` before recording so Bluetooth headset input does not unnecessarily hijack playback when a built-in mic fallback is available.
- `ParakeetEngine` consults `ParakeetStartRecordingFailurePolicy` when startup fails so format-reset, engine rebuild, and prewarm-retry behavior stay consistent across direct starts and recovery attempts.
- Route-change analytics are committed only after the config-change debounce resolves to a new categorical route. Notification churn and A -> B -> A oscillation stay quiet, but the underlying device-recovery state machine still runs.
- Zombie-engine recovery owns a separate generation-gated task, replaces the stale `AVAudioEngine` through a timed reset, and retries once. Do not fold that task back into the startup watchdog or reuse the detected zombie graph.
- `ParakeetEngine` stays `@MainActor` for app state, published UI state, and event reporting, but all `AVAudioEngine` graph work runs through its private serial audio-engine queue. Keep recording start/stop/readiness APIs async so callers do not block the main actor while CoreAudio settles, starts, stops, or rebuilds.
- `ParakeetEngine` reports model-init failures with `ParakeetModelInitDiagnostics.failureContext(...)`, which keeps diagnostics useful for packaging/download/debugging issues without shipping raw transcript or device content.
- Dictation intentionally exposes only final transcription. The abandoned provisional-text/EOU path was removed rather than kept behind a false feature flag. A future live dictation experience must add a real streaming engine and end-to-end tests instead of reviving dormant audio-tap branches.
Expand Down
100 changes: 64 additions & 36 deletions Sources/Speech/ParakeetDeviceRecovery.swift
Original file line number Diff line number Diff line change
Expand Up @@ -83,16 +83,8 @@ extension ParakeetEngine {
}

private func handleDefaultInputDeviceChange(selection: DictationInputDeviceSelection) {
cachedInputDeviceName = selection.selectedInput.name
cachedInputDeviceSelection = selection
AppLogger.transcription.info("PARAKEET | default input changed → \(selection.defaultInput.name); dictation input → \(selection.selectedInput.name)")
EventReporter.shared.capture(
level: .info,
engine: "parakeet",
event: "default_input_device_changed",
message: "Default input device changed",
context: inputSelectionContext(selection)
)
routeTransitionDebounceState.observe(categoricalAudioRoute(for: selection))
updateCachedInputDeviceSelection(selection)
Task { @MainActor [weak self] in
await self?.handleAudioConfigChange()
}
Expand Down Expand Up @@ -129,16 +121,6 @@ extension ParakeetEngine {
cancelConfigRecoveryTimeout()
let recoveryGeneration = recoveryState.beginConfigChange()
publishRecoveryState()
// Load the new route off the main actor — the enumeration is blocking
// coreaudiod IPC — then refresh the analytics cache and report.
let wasRecordingForAnalytics = configChangeWasRecording
Task.detached(priority: .utility) { [weak self] in
let selection = Self.loadDictationInputDeviceSelection()
await self?.recordRouteChangeAnalytics(
selection: selection,
wasRecording: wasRecordingForAnalytics
)
}
scheduleConfigRecoveryTimeout(
generation: recoveryGeneration,
wasRecording: configChangeWasRecording
Expand All @@ -152,17 +134,21 @@ extension ParakeetEngine {
cancelAudioWatchdog()
prewarmRetryTask?.cancel()
prewarmRetryTask = nil
let configCleanupOwner = currentAudioEngineQueueOwnerToken()

if isRecording {
preserveCurrentRecordingBuffersForRecovery()
await removeRecordingTap()
guard ownsAudioEngineQueue(configCleanupOwner) else { return }
isRecording = false
audioLevel = 0
}

await stopAudioEngine()
guard ownsAudioEngineQueue(configCleanupOwner) else { return }
isEnginePrewarmed = false
await rebuildAudioEngine(reason: "configuration_change")
guard let rebuiltOwner = await rebuildAudioEngine(reason: "configuration_change"),
ownsAudioGraph(rebuiltOwner) else { return }

// Cancel any in-flight recovery — the latest device change wins.
// Bluetooth disconnect/reconnect fires multiple notifications over
Expand All @@ -176,25 +162,50 @@ extension ParakeetEngine {
// short enough that dictation recovery feels responsive.
try? await Task.sleep(nanoseconds: TranscriptedConstants.audioConfigChangeDebounceDelay)
guard !Task.isCancelled, let self = self else { return }
let wasRecordingForAnalytics = self.configChangeWasRecording
Task.detached(priority: .utility) { [weak self] in
let selection = Self.loadDictationInputDeviceSelection()
await self?.recordStableRouteChangeAnalytics(
selection: selection,
wasRecording: wasRecordingForAnalytics,
recoveryGeneration: recoveryGeneration
)
}
// Telemetry coalescing must never suppress the real recovery state
// transition, including an A -> B -> A notification burst.
self.attemptDeviceRecovery()
}
}

private func recordRouteChangeAnalytics(
private func recordStableRouteChangeAnalytics(
selection: DictationInputDeviceSelection?,
wasRecording: Bool
wasRecording: Bool,
recoveryGeneration: UInt64
) {
if let selection {
updateCachedInputDeviceSelection(selection)
guard !recoveryState.isStale(generation: recoveryGeneration) else { return }
guard let selection else {
routeTransitionDebounceState.discardPendingRoute()
return
}
routeTransitionDebounceState.observe(categoricalAudioRoute(for: selection))
updateCachedInputDeviceSelection(selection)
guard let stableRoute = routeTransitionDebounceState.commitPendingRoute() else { return }

AppLogger.transcription.info("PARAKEET | stable input route changed → \(stableRoute.routeShape)")
let context = dictationRouteAnalyticsContext(
selection: selection,
extra: ["was_recording": "\(wasRecording)"]
)
EventReporter.shared.capture(
level: .info,
engine: "parakeet",
event: "default_input_device_changed",
message: "Stable categorical input route changed",
context: context
)
AnalyticsReporter.track(
"dictation_audio_route_changed",
properties: dictationRouteAnalyticsContext(
selection: selection,
extra: [
"was_recording": "\(wasRecording)"
]
)
properties: context
)
}

Expand Down Expand Up @@ -274,11 +285,13 @@ extension ParakeetEngine {
return
}

var lastSnapshotOwner: ParakeetAudioEngineQueueOwnerToken?
do {
var recoveryAttempt = 0
var readySnapshot: ParakeetAudioInputSnapshot?
while readySnapshot == nil {
recoveryAttempt += 1
lastSnapshotOwner = self.currentAudioEngineQueueOwnerToken()
let snapshot = try await self.audioInputSnapshot(
operation: recoveryAttempt == 1 ? "device_recovery" : "device_recovery_retry",
recoveryGeneration: myGeneration
Expand Down Expand Up @@ -344,7 +357,10 @@ extension ParakeetEngine {
for attempt in 1...TranscriptedConstants.recordingRestartAttempts {
guard !Task.isCancelled else { return }
guard !self.recoveryState.isStale(generation: myGeneration) else { return }
if await self.startRecording() {
let startSucceeded = await self.startRecording()
guard !Task.isCancelled else { return }
guard !self.recoveryState.isStale(generation: myGeneration) else { return }
if startSucceeded {
restarted = true
AppLogger.transcription.info("PARAKEET | recording recovered on new device (\(self.inputDeviceName)) after \(attempt) attempt(s)")
EventReporter.shared.capture(level: .info, engine: "parakeet",
Expand Down Expand Up @@ -387,6 +403,10 @@ extension ParakeetEngine {
// / Bluetooth route-switch hang). Rebuilding on that same queue would
// never run, so fail safe by abandoning the blocked graph instead.
let audioEngineQueueBlocked = error is ParakeetAudioEngineWorkError
if audioEngineQueueBlocked {
guard let lastSnapshotOwner,
self.ownsAudioEngineQueue(lastSnapshotOwner) else { return }
}
let failureAction = ParakeetDeviceRecoveryFailurePolicy.action(wasRecording: shouldRestartRecording)
if self.recoveryState.finishRecovery(success: false, generation: myGeneration) {
self.cancelConfigRecoveryTimeout()
Expand Down Expand Up @@ -446,9 +466,13 @@ extension ParakeetEngine {
audioEngineQueueBlocked: audioEngineQueueBlocked
) {
case .queuedOnAudioEngineQueue:
await self.rebuildAudioEngine(reason: "device_change_rewarm_failed")
guard await self.rebuildAudioEngine(reason: "device_change_rewarm_failed") != nil else { return }
case .abandonBlockedAudioGraph:
self.abandonBlockedAudioEngine(reason: "device_change_rewarm_failed")
guard let lastSnapshotOwner,
self.abandonBlockedAudioEngine(
reason: "device_change_rewarm_failed",
expectedOwner: lastSnapshotOwner
) else { return }
}
if failureAction.schedulePrewarmRetry {
self.prewarmRetryCount = 0
Expand Down Expand Up @@ -528,11 +552,15 @@ extension ParakeetEngine {
)
)
}
let timeoutOwner = self.currentAudioEngineQueueOwnerToken()
switch timeoutAction.rebuildStrategy {
case .queuedOnAudioEngineQueue:
await self.rebuildAudioEngine(reason: "device_change_recovery_timeout")
guard await self.rebuildAudioEngine(reason: "device_change_recovery_timeout") != nil else { return }
case .abandonBlockedAudioGraph:
self.abandonBlockedAudioEngine(reason: "device_change_recovery_timeout")
guard self.abandonBlockedAudioEngine(
reason: "device_change_recovery_timeout",
expectedOwner: timeoutOwner
) else { return }
}
if failureAction.schedulePrewarmRetry {
self.prewarmRetryCount = 0
Expand Down
Loading
Loading