diff --git a/Sources/Dictation/CLAUDE.md b/Sources/Dictation/CLAUDE.md index aca035292..3b8fc50c8 100644 --- a/Sources/Dictation/CLAUDE.md +++ b/Sources/Dictation/CLAUDE.md @@ -7,6 +7,7 @@ ## Files - `DictationSessionTimeout.swift` — uptime-based timeout helper so sleep does not consume a session's remaining record window +- `DictationStoppedAudioRecovery.swift` — writes a private recovery WAV plus restart-discovery metadata immediately after recording stops and retains both until transcript persistence succeeds or the user explicitly discards the session - `DictationStoragePaths.swift` — capture-library-backed storage root for dictation artifacts - `DictationTranscriptWriter.swift` — groups completed dictations into one markdown file per day; serializes day-file writes through `DictationTranscriptMutationLock` - `DictationTranscriptStore.swift` — shared seam for saving dictation markdown and reading the newest saved dictation back out @@ -27,6 +28,13 @@ - root: `/dictations/` - transcript folder: same as the dictation root - file shape: one `Dictations_YYYY-MM-DD.md` file per day, with multiple timestamped sections +- stopped-audio recovery: `~/Library/Application Support/Transcripted/state/dictation-audio-recovery/` + +Stopped-audio recovery is intentionally bounded and local. Launch scans at most +one pending metadata record for presentation, then `Show Audio` reveals the WAV +in Finder. The operational recovery path is Home -> Import Audio -> select that +WAV; this uses the normal local imported-audio transcription pipeline. Reveal or +restart never deletes the checkpoint. Each section captures: @@ -40,6 +48,7 @@ Each section captures: ## Test coverage - `Tests/DictationSessionTimeoutTests.swift` +- `Tests/DictationStoppedAudioRecoveryTests.swift` - `Tests/DictationTranscriptStoreTests.swift` - `Tests/DictationTranscriptWriterTests.swift` diff --git a/Sources/Dictation/DictationStoppedAudioRecovery.swift b/Sources/Dictation/DictationStoppedAudioRecovery.swift new file mode 100644 index 000000000..062fca195 --- /dev/null +++ b/Sources/Dictation/DictationStoppedAudioRecovery.swift @@ -0,0 +1,186 @@ +// DictationStoppedAudioRecovery.swift +// Durable audio checkpoint for a stopped dictation awaiting transcription. + +import Foundation + +struct DictationStoppedAudioRecovery: Equatable, Sendable { + let url: URL + let sessionID: UUID + let createdAt: Date +} + +enum DictationStoppedAudioRecoveryCommitPolicy { + static func shouldPersist( + taskCancelled: Bool, + isDictating: Bool, + taskSessionID: UUID, + currentSessionID: UUID + ) -> Bool { + !taskCancelled && isDictating && taskSessionID == currentSessionID + } + + static func shouldRetainPersistedRecovery( + taskSessionID: UUID, + preservationSessionID: UUID? + ) -> Bool { + taskSessionID == preservationSessionID + } +} + +enum DictationStoppedAudioRecoveryStore { + static let sampleRate: UInt32 = 16_000 + + private struct Metadata: Codable { + let version: Int + let sessionID: UUID + let createdAt: Date + let audioFilename: String + } + + static var defaultDirectory: URL { + FileManager.default.transcriptedStateDir + .appendingPathComponent("dictation-audio-recovery", isDirectory: true) + } + + static func persist( + samples16k: [Float], + sessionID: UUID, + createdAt: Date = Date(), + directory: URL? = nil, + fileManager: FileManager = .default + ) throws -> DictationStoppedAudioRecovery? { + guard !samples16k.isEmpty else { return nil } + + let folder = directory ?? defaultDirectory + try fileManager.createPrivateDirectory(at: folder) + let url = folder.appendingPathComponent("dictation_\(sessionID.uuidString.lowercased()).wav") + try wavData(samples16k: samples16k).write(to: url, options: .atomic) + fileManager.restrictFileToOwnerOnly(at: url) + let recovery = DictationStoppedAudioRecovery(url: url, sessionID: sessionID, createdAt: createdAt) + do { + try writeMetadata(for: recovery, fileManager: fileManager) + } catch { + try? fileManager.removeItem(at: url) + throw error + } + return recovery + } + + static func pendingRecoveries( + limit: Int = 10, + directory: URL? = nil, + fileManager: FileManager = .default + ) -> [DictationStoppedAudioRecovery] { + guard limit > 0 else { return [] } + let folder = directory ?? defaultDirectory + guard let enumerator = fileManager.enumerator( + at: folder, + includingPropertiesForKeys: [.isRegularFileKey], + options: [.skipsHiddenFiles, .skipsSubdirectoryDescendants] + ) else { return [] } + + var recoveries: [DictationStoppedAudioRecovery] = [] + for case let metadataURL as URL in enumerator where metadataURL.pathExtension == "json" { + guard let metadata = try? JSONDecoder().decode(Metadata.self, from: Data(contentsOf: metadataURL)), + metadata.version == 1 else { continue } + let audioURL = folder.appendingPathComponent(metadata.audioFilename, isDirectory: false) + guard fileManager.fileExists(atPath: audioURL.path) else { continue } + recoveries.append(DictationStoppedAudioRecovery( + url: audioURL, + sessionID: metadata.sessionID, + createdAt: metadata.createdAt + )) + } + return mostRecent(recoveries, limit: limit) + } + + static func mostRecent( + _ recoveries: [DictationStoppedAudioRecovery], + limit: Int + ) -> [DictationStoppedAudioRecovery] { + guard limit > 0 else { return [] } + return Array( + recoveries + .sorted { $0.createdAt > $1.createdAt } + .prefix(limit) + ) + } + + @discardableResult + static func cleanup( + _ recovery: DictationStoppedAudioRecovery?, + transcriptPersisted: Bool = false, + explicitDiscard: Bool = false, + fileManager: FileManager = .default + ) -> Bool { + guard transcriptPersisted || explicitDiscard, + let recovery else { return false } + + do { + if fileManager.fileExists(atPath: recovery.url.path) { + try fileManager.removeItem(at: recovery.url) + } + let metadataURL = metadataURL(for: recovery.url) + if fileManager.fileExists(atPath: metadataURL.path) { + try fileManager.removeItem(at: metadataURL) + } + return true + } catch { + return false + } + } + + private static func writeMetadata( + for recovery: DictationStoppedAudioRecovery, + fileManager: FileManager + ) throws { + let metadata = Metadata( + version: 1, + sessionID: recovery.sessionID, + createdAt: recovery.createdAt, + audioFilename: recovery.url.lastPathComponent + ) + let url = metadataURL(for: recovery.url) + try JSONEncoder().encode(metadata).write(to: url, options: .atomic) + fileManager.restrictFileToOwnerOnly(at: url) + } + + private static func metadataURL(for audioURL: URL) -> URL { + audioURL.deletingPathExtension().appendingPathExtension("json") + } + + private static func wavData(samples16k: [Float]) -> Data { + let bytesPerSample: UInt16 = 2 + let channelCount: UInt16 = 1 + let dataByteCount = UInt32(samples16k.count) * UInt32(bytesPerSample) + var data = Data(capacity: 44 + Int(dataByteCount)) + + data.append(contentsOf: "RIFF".utf8) + append(UInt32(36) + dataByteCount, to: &data) + data.append(contentsOf: "WAVEfmt ".utf8) + append(UInt32(16), to: &data) + append(UInt16(1), to: &data) + append(channelCount, to: &data) + append(sampleRate, to: &data) + append(sampleRate * UInt32(channelCount) * UInt32(bytesPerSample), to: &data) + append(channelCount * bytesPerSample, to: &data) + append(UInt16(16), to: &data) + data.append(contentsOf: "data".utf8) + append(dataByteCount, to: &data) + + for sample in samples16k { + let finiteSample = sample.isFinite ? sample : 0 + let clamped = max(-1, min(1, finiteSample)) + let pcm = Int16((clamped * Float(Int16.max)).rounded()) + append(UInt16(bitPattern: pcm), to: &data) + } + return data + } + + private static func append(_ value: T, to data: inout Data) { + var littleEndian = value.littleEndian + withUnsafeBytes(of: &littleEndian) { bytes in + data.append(contentsOf: bytes) + } + } +} diff --git a/Sources/Meeting/MeetingSessionController.swift b/Sources/Meeting/MeetingSessionController.swift index 806894434..58fd189f1 100644 --- a/Sources/Meeting/MeetingSessionController.swift +++ b/Sources/Meeting/MeetingSessionController.swift @@ -202,6 +202,7 @@ final class MeetingSessionController: ObservableObject { var liveCodexFinalTranscriptNeedsQueuedJobID = false var liveCodexAwaitedTranscriptionJobID: UUID? var activeQueuedTranscriptionJobID: UUID? + var activeStoppedAudioRecovery: DictationStoppedAudioRecovery? var shouldConfirmQuitForActiveCapture: Bool { isCaptureSessionActive || isFinishingRecording @@ -1453,11 +1454,15 @@ final class MeetingSessionController: ObservableObject { return false } + let stoppedAudioRecovery = DictationStoppedAudioRecoveryStore + .pendingRecoveries(limit: Int.max) + .first { $0.url.standardizedFileURL == sourceURL.standardizedFileURL } let outcome = transcriptionQueue.enqueueImportedAudioJob( audioURL: preparedAudio.copiedAudioURL, suggestedTitle: preparedAudio.suggestedTitle, recordingDate: preparedAudio.recordingDate, - startTrigger: .fileImport + startTrigger: .fileImport, + stoppedAudioRecovery: stoppedAudioRecovery ) DiagnosticsTrail.record( @@ -1541,8 +1546,9 @@ final class MeetingSessionController: ObservableObject { taskManager.cancelAll() if liveCodexSessionAwaitingFinalTranscript { finishLiveCodexSession(status: .failed, shouldAwaitFinalTranscript: false) - activeQueuedTranscriptionJobID = nil } + activeQueuedTranscriptionJobID = nil + activeStoppedAudioRecovery = nil state = .ready DiagnosticsTrail.record( level: .warning, @@ -2589,6 +2595,15 @@ final class MeetingSessionController: ObservableObject { switch status { case .transcriptSaved: lastTerminalTranscriptionOutcome = .transcriptSaved + if let stoppedAudioRecovery = activeStoppedAudioRecovery { + activeStoppedAudioRecovery = nil + Task.detached(priority: .utility) { + DictationStoppedAudioRecoveryStore.cleanup( + stoppedAudioRecovery, + transcriptPersisted: true + ) + } + } let transcriptionTrigger = activeTranscriptionTrigger let promptTelemetryProperties = activeDetectedPromptTranscriptionTelemetryProperties let promptRecordingStartedAt = activeDetectedPromptTranscriptionRecordingStartedAt @@ -2618,6 +2633,8 @@ final class MeetingSessionController: ObservableObject { activeTranscriptionCaptureDiagnostics = nil case .failed(let message): lastTerminalTranscriptionOutcome = .failed(message) + // A failed import must retain its original stopped-audio checkpoint. + activeStoppedAudioRecovery = nil let transcriptionTrigger = activeTranscriptionTrigger let diagnosticMessage = taskManager.lastFailureDiagnosticMessage ?? message let failureKind = MeetingFailureKind.classify(message: diagnosticMessage) diff --git a/Sources/Meeting/TranscriptionQueueCoordinator.swift b/Sources/Meeting/TranscriptionQueueCoordinator.swift index 0675c5e57..c7e8dd0c3 100644 --- a/Sources/Meeting/TranscriptionQueueCoordinator.swift +++ b/Sources/Meeting/TranscriptionQueueCoordinator.swift @@ -40,6 +40,7 @@ final class TranscriptionQueueCoordinator { let kind: Kind let startTrigger: MeetingSessionController.StartTrigger let sttModel: TranscriptionModelChoice + let stoppedAudioRecovery: DictationStoppedAudioRecovery? let promptTelemetryProperties: [String: String]? let promptRecordingStartedAt: Date? @@ -120,6 +121,7 @@ final class TranscriptionQueueCoordinator { ), startTrigger: startTrigger, sttModel: controller.sttRouter.selectedModel, + stoppedAudioRecovery: nil, promptTelemetryProperties: promptTelemetryProperties, promptRecordingStartedAt: promptRecordingStartedAt ) @@ -135,7 +137,8 @@ final class TranscriptionQueueCoordinator { audioURL: URL, suggestedTitle: String, recordingDate: Date, - startTrigger: MeetingSessionController.StartTrigger + startTrigger: MeetingSessionController.StartTrigger, + stoppedAudioRecovery: DictationStoppedAudioRecovery? = nil ) -> QueueInsertionOutcome { let job = QueuedTranscriptionJob( kind: .imported( @@ -145,6 +148,7 @@ final class TranscriptionQueueCoordinator { ), startTrigger: startTrigger, sttModel: controller.sttRouter.selectedModel, + stoppedAudioRecovery: stoppedAudioRecovery, promptTelemetryProperties: nil, promptRecordingStartedAt: nil ) @@ -298,6 +302,7 @@ final class TranscriptionQueueCoordinator { controller.sttAdapter.selectPreparedModel(job.sttModel) queuedRuntimeDiagnosticsJobIDs.remove(job.id) controller.activeQueuedTranscriptionJobID = job.id + controller.activeStoppedAudioRecovery = job.stoppedAudioRecovery switch job.kind { case .recorded(let micURL, let systemURL, let healthInfo, _, let meetingTitle, let recordingDate): diff --git a/Sources/Speech/ParakeetEngine.swift b/Sources/Speech/ParakeetEngine.swift index 9679c9019..7a5c1df68 100644 --- a/Sources/Speech/ParakeetEngine.swift +++ b/Sources/Speech/ParakeetEngine.swift @@ -2324,6 +2324,49 @@ class ParakeetEngine: ObservableObject { return (nativeSampleCount, resampled) } + private func consumeRecordedSamples( + preparedRecording: RecordedSpeechSamples? + ) async -> (nativeSampleCount: Int, samples16k: [Float])? { + guard let preparedRecording else { + return await drainRecordedSamplesForInference() + } + + // The persistence snapshot already resampled this exact stopped + // recording. Consume the native buffers without repeating that work. + drainPendingSamplesIntoSampleBuffer() + sampleBuffer.removeAll(keepingCapacity: true) + clearRecoveredRecordingTimeline(keepingCapacity: true) + return ( + nativeSampleCount: preparedRecording.nativeSampleCount, + samples16k: preparedRecording.samples16k + ) + } + + func snapshotRecordedSamplesForPersistence() async -> RecordedSpeechSamples? { + drainPendingSamplesIntoSampleBuffer() + + var segments = recoveredRecordingTimeline.segments + if !sampleBuffer.isEmpty { + segments.append(RecordedAudioSegment(sampleRate: safeNativeSampleRate(), samples: sampleBuffer)) + } + let nativeSampleCount = segments.reduce(0) { $0 + $1.samples.count } + guard nativeSampleCount > 0 else { return nil } + + let samples16k = await Task.detached(priority: .userInitiated) { + var combined: [Float] = [] + combined.reserveCapacity(nativeSampleCount) + for segment in segments { + combined.append(contentsOf: AudioResampler.resample( + segment.samples, + from: segment.sampleRate, + to: TranscriptedConstants.parakeetSampleRate + )) + } + return combined + }.value + return RecordedSpeechSamples(nativeSampleCount: nativeSampleCount, samples16k: samples16k) + } + /// Convert [Float] samples to AVAudioPCMBuffer for StreamingEouAsrManager. private func makePCMBuffer(from samples: [Float]) -> AVAudioPCMBuffer? { guard let format = eouPCMFormat, @@ -2341,7 +2384,10 @@ class ParakeetEngine: ObservableObject { // MARK: - Transcription - func drainRecordedSamplesForExternalTranscription(engineName: String) async -> RecordedSpeechSamples? { + func drainRecordedSamplesForExternalTranscription( + engineName: String, + preparedRecording: RecordedSpeechSamples? = nil + ) async -> RecordedSpeechSamples? { lastEmptyTranscriptionReason = nil guard !isTranscribing else { EventReporter.shared.capture( @@ -2355,7 +2401,7 @@ class ParakeetEngine: ObservableObject { drainPendingSamplesIntoSampleBuffer() - guard !sampleBuffer.isEmpty || !recoveredRecordingTimeline.isEmpty else { + guard preparedRecording != nil || !sampleBuffer.isEmpty || !recoveredRecordingTimeline.isEmpty else { lastEmptyTranscriptionReason = .recordingTooShort EventReporter.shared.capture( level: .warning, @@ -2367,7 +2413,7 @@ class ParakeetEngine: ObservableObject { } isTranscribing = true - guard let recorded = await drainRecordedSamplesForInference() else { + guard let recorded = await consumeRecordedSamples(preparedRecording: preparedRecording) else { finishExternalTranscription() return nil } @@ -2481,7 +2527,7 @@ class ParakeetEngine: ObservableObject { } } - func transcribe() async -> String? { + func transcribe(preparedRecording: RecordedSpeechSamples? = nil) async -> String? { lastEmptyTranscriptionReason = nil guard !isTranscribing else { EventReporter.shared.capture(level: .warning, engine: "parakeet", event: "transcription_already_active", @@ -2489,7 +2535,7 @@ class ParakeetEngine: ObservableObject { return nil } drainPendingSamplesIntoSampleBuffer() - guard !sampleBuffer.isEmpty || !recoveredRecordingTimeline.isEmpty else { + guard preparedRecording != nil || !sampleBuffer.isEmpty || !recoveredRecordingTimeline.isEmpty else { lastEmptyTranscriptionReason = .recordingTooShort EventReporter.shared.capture(level: .warning, engine: "parakeet", event: "no_audio_samples", message: "No audio samples in buffer when transcribe() called") @@ -2505,7 +2551,7 @@ class ParakeetEngine: ObservableObject { isTranscribing = true let startTime = CFAbsoluteTimeGetCurrent() - guard let recorded = await drainRecordedSamplesForInference() else { + guard let recorded = await consumeRecordedSamples(preparedRecording: preparedRecording) else { finishTranscription() return nil } @@ -2602,6 +2648,16 @@ class ParakeetEngine: ObservableObject { emptyContext["retry_samples"] = "\(retrySamples.count)" } catch { emptyContext["retry_error"] = error.localizedDescription + EventReporter.shared.capture( + level: .error, + engine: "parakeet", + event: "dictation_empty_retry_failed", + message: "Focused retry failed after the initial empty transcription", + context: emptyContext + ) + lastEmptyTranscriptionReason = .modelFailure + finishTranscription() + return nil } } else if !analysis.hasUsableSpeechSignal { EventReporter.shared.capture( diff --git a/Sources/Speech/STTRouter.swift b/Sources/Speech/STTRouter.swift index 90c67fe12..f892ead69 100644 --- a/Sources/Speech/STTRouter.swift +++ b/Sources/Speech/STTRouter.swift @@ -197,6 +197,10 @@ class STTRouter: ObservableObject { await parakeetEngine.stopRecording() } + func snapshotRecordedSamplesForPersistence() async -> RecordedSpeechSamples? { + await parakeetEngine.snapshotRecordedSamplesForPersistence() + } + func resetAfterFailedRecordingStart() async { activeRecordingModel = nil await parakeetEngine.resetAfterFailedRecordingStart() @@ -207,7 +211,7 @@ class STTRouter: ObservableObject { parakeetEngine.abandonBlockedRecordingStart(reason: reason) } - func transcribe() async -> String? { + func transcribe(preparedRecording: RecordedSpeechSamples? = nil) async -> String? { let model = activeRecordingModel ?? selectedModel lastEmptyTranscriptionReason = nil defer { @@ -216,11 +220,14 @@ class STTRouter: ObservableObject { switch model { case .parakeetTDTv3: - let text = await parakeetEngine.transcribe() + let text = await parakeetEngine.transcribe(preparedRecording: preparedRecording) lastEmptyTranscriptionReason = text == nil ? parakeetEngine.lastEmptyTranscriptionReason : nil return text case .whisperLargeV3Turbo, .whisperLargeV3: - return await transcribeUsingExternalEngine(model: model) { [self] recording in + return await transcribeUsingExternalEngine( + model: model, + preparedRecording: preparedRecording + ) { [self] recording in try await whisperEngine.transcribeSamples( recording.samples16k, source: .microphone, @@ -228,7 +235,10 @@ class STTRouter: ObservableObject { ) } case .nemotronStreaming: - return await transcribeUsingExternalEngine(model: model) { [self] recording in + return await transcribeUsingExternalEngine( + model: model, + preparedRecording: preparedRecording + ) { [self] recording in try await nemotronEngine.transcribeSamples( recording.samples16k, source: .microphone @@ -242,10 +252,12 @@ class STTRouter: ObservableObject { /// Parakeet samples rather than owning the audio graph themselves. private func transcribeUsingExternalEngine( model: TranscriptionModelChoice, + preparedRecording: RecordedSpeechSamples?, transcribe: (RecordedSpeechSamples) async throws -> String ) async -> String? { await initialize(model: model) guard isModelLoaded(for: model) else { + lastEmptyTranscriptionReason = .modelFailure EventReporter.shared.capture( level: .error, engine: model.engineName, @@ -257,7 +269,8 @@ class STTRouter: ObservableObject { } guard let recording = await parakeetEngine.drainRecordedSamplesForExternalTranscription( - engineName: model.engineName + engineName: model.engineName, + preparedRecording: preparedRecording ) else { lastEmptyTranscriptionReason = parakeetEngine.lastEmptyTranscriptionReason return nil @@ -275,6 +288,7 @@ class STTRouter: ObservableObject { } return text } catch { + lastEmptyTranscriptionReason = .modelFailure EventReporter.shared.capture( level: .error, engine: model.engineName, diff --git a/Sources/TranscriptedApp.swift b/Sources/TranscriptedApp.swift index b9184b063..625f59275 100644 --- a/Sources/TranscriptedApp.swift +++ b/Sources/TranscriptedApp.swift @@ -145,6 +145,7 @@ class TranscriptedAppDelegate: NSObject, NSApplicationDelegate, NSPopoverDelegat // Set up the floating overlay panel (pure AppKit — no NSHostingView) overlayController.setup(sttRouter: appState.sttRouter) + sessionController.presentPendingStoppedAudioRecoveryIfNeeded() // Meeting overlay + hotkey + speaker naming — Lane C wiring. if #available(macOS 14.0, *) { diff --git a/Sources/UI/Overlay/DictationSessionController.swift b/Sources/UI/Overlay/DictationSessionController.swift index 6678f15bb..6099a0366 100644 --- a/Sources/UI/Overlay/DictationSessionController.swift +++ b/Sources/UI/Overlay/DictationSessionController.swift @@ -5,6 +5,26 @@ import AppKit import AVFoundation import Combine +private actor DictationStoppedAudioCheckpointSignal { + private var isComplete = false + private var waiters: [CheckedContinuation] = [] + + func wait() async { + guard !isComplete else { return } + await withCheckedContinuation { continuation in + waiters.append(continuation) + } + } + + func complete() { + guard !isComplete else { return } + isComplete = true + let pendingWaiters = waiters + waiters.removeAll() + pendingWaiters.forEach { $0.resume() } + } +} + @MainActor class DictationSessionController: ObservableObject { enum DictationTrigger: String { @@ -65,6 +85,9 @@ class DictationSessionController: ObservableObject { private var sessionStartTime: CFAbsoluteTime = 0 private var currentDictationTrigger: DictationTrigger = .unknown private var currentDictationSessionID = UUID() + private var stoppedAudioRecovery: DictationStoppedAudioRecovery? + private var stoppedAudioRecoveryPreservationSessionID: UUID? + private var stoppedAudioCheckpointSignal: DictationStoppedAudioCheckpointSignal? private var autoSendRequestDecision = DictationAutoSendRequestDecision.notEvaluated /// Max duration for a listening session before auto-cancel (5 minutes). @@ -97,6 +120,19 @@ class DictationSessionController: ObservableObject { } } + func presentPendingStoppedAudioRecoveryIfNeeded() { + guard !isDictating, + let overlayController, + let recovery = DictationStoppedAudioRecoveryStore.pendingRecoveries(limit: 1).first else { return } + overlayController.showError( + "A stopped dictation recording is available. Use Import Audio from Home to recover its transcript.", + actionTitle: "Show Audio", + action: { + NSWorkspace.shared.activateFileViewerSelecting([recovery.url]) + } + ) + } + // MARK: - Dictation Mode (Option+Space) /// Start dictation — show overlay and begin voice recording (no screenshot/vision) @@ -118,6 +154,9 @@ class DictationSessionController: ObservableObject { } isDictating = true currentDictationSessionID = UUID() + stoppedAudioRecovery = nil + stoppedAudioRecoveryPreservationSessionID = nil + stoppedAudioCheckpointSignal = nil sessionSourceApp = sourceApp sessionPasteTarget = DictationPasteTarget.capture(sourceApp: sourceApp) sessionAnchorRect = anchorRect @@ -907,8 +946,14 @@ class DictationSessionController: ObservableObject { streamingTask?.cancel() let taskSessionID = currentDictationSessionID + let checkpointSignal = DictationStoppedAudioCheckpointSignal() + stoppedAudioCheckpointSignal = checkpointSignal streamingTask = Task { + defer { + Task { await checkpointSignal.complete() } + } var stopTiming = DictationStopTiming(requestedAt: stopRequestedAt) + var stoppedRecordingSnapshot: RecordedSpeechSamples? appState.runtimeDiagnostics.recordSession(kind: "dictation", stage: "stop_requested") if appState.sttRouter.isRecording || appState.sttRouter.hasRecoverableRecording { await appState.sttRouter.stopRecording() @@ -918,6 +963,64 @@ class DictationSessionController: ObservableObject { self.isDictating, self.currentDictationSessionID == taskSessionID else { return } + do { + if let recording = await appState.sttRouter.snapshotRecordedSamplesForPersistence() { + guard DictationStoppedAudioRecoveryCommitPolicy.shouldPersist( + taskCancelled: Task.isCancelled, + isDictating: self.isDictating, + taskSessionID: taskSessionID, + currentSessionID: self.currentDictationSessionID + ) else { return } + let recovery = try await Task.detached(priority: .userInitiated) { + try DictationStoppedAudioRecoveryStore.persist( + samples16k: recording.samples16k, + sessionID: taskSessionID + ) + }.value + guard DictationStoppedAudioRecoveryCommitPolicy.shouldPersist( + taskCancelled: Task.isCancelled, + isDictating: self.isDictating, + taskSessionID: taskSessionID, + currentSessionID: self.currentDictationSessionID + ) else { + if !DictationStoppedAudioRecoveryCommitPolicy.shouldRetainPersistedRecovery( + taskSessionID: taskSessionID, + preservationSessionID: self.stoppedAudioRecoveryPreservationSessionID + ) { + await Task.detached(priority: .utility) { + DictationStoppedAudioRecoveryStore.cleanup( + recovery, + explicitDiscard: true + ) + }.value + } + return + } + self.stoppedAudioRecovery = recovery + stoppedRecordingSnapshot = recording + } + } catch { + guard DictationStoppedAudioRecoveryCommitPolicy.shouldPersist( + taskCancelled: Task.isCancelled, + isDictating: self.isDictating, + taskSessionID: taskSessionID, + currentSessionID: self.currentDictationSessionID + ) else { return } + appState.logger.log("DICTATION | failed to preserve stopped audio: \(error.localizedDescription)") + EventReporter.shared.capture( + level: .error, + engine: "dictation", + event: "dictation_stopped_audio_persistence_failed", + message: error.localizedDescription + ) + overlayController.showError("The recording stopped, but its audio couldn't be saved safely. Free some disk space and try again.") + self.isDictating = false + appState.runtimeDiagnostics.clearSession(kind: "dictation", outcome: "audio_persistence_failed") + return + } + + await checkpointSignal.complete() + // Surface model warmup honestly instead of calling it "Transcribing" // before the local dictation model is actually ready. if !appState.sttRouter.isModelLoaded { @@ -979,7 +1082,9 @@ class DictationSessionController: ObservableObject { overlayController.resizePanelToCompact() appState.runtimeDiagnostics.recordSession(kind: "dictation", stage: "transcribing") stopTiming.transcriptionStartedAt = CFAbsoluteTimeGetCurrent() - let voiceText = await appState.sttRouter.transcribe() + let voiceText = await appState.sttRouter.transcribe( + preparedRecording: stoppedRecordingSnapshot + ) stopTiming.transcribedAt = CFAbsoluteTimeGetCurrent() guard !Task.isCancelled, self.isDictating, @@ -1097,6 +1202,7 @@ class DictationSessionController: ObservableObject { ) let autoSendOutcome = finalization.autoEnterOutcome let saveResult = finalization.saveResult + self.discardStoppedAudioRecovery(transcriptPersisted: saveResult.saved != nil) let saveFailureMessage = saveResult.failureMessage let wordCount = text.split(whereSeparator: \.isWhitespace).count stopTiming.completedAt = CFAbsoluteTimeGetCurrent() @@ -1237,6 +1343,7 @@ class DictationSessionController: ObservableObject { ) { lastCompletedText = text let saveResult = persistDictationTranscript(text: text, delivery: .savedWithoutPaste) + discardStoppedAudioRecovery(transcriptPersisted: saveResult.saved != nil) let saveFailureMessage = saveResult.failureMessage let wordCount = text.split(whereSeparator: \.isWhitespace).count let durationSeconds = CFAbsoluteTimeGetCurrent() - sessionStartTime @@ -1314,9 +1421,15 @@ class DictationSessionController: ObservableObject { } /// Cancel dictation without pasting - func cancelDictation() { + func cancelDictation(preserveStoppedAudio: Bool = false) { guard let (appState, overlayController) = readyState() else { return } + if preserveStoppedAudio { + stoppedAudioRecoveryPreservationSessionID = currentDictationSessionID + } cancelActiveTasks(cancelRecording: true) + if !preserveStoppedAudio { + discardStoppedAudioRecovery(explicitDiscard: true) + } AppSoundPlayer.shared.play(.dictationCancelled) overlayController.hideWithCancelAnimation() isDictating = false @@ -1368,7 +1481,11 @@ class DictationSessionController: ObservableObject { } if isDictating { - cancelDictation() + stoppedAudioRecoveryPreservationSessionID = currentDictationSessionID + if let stoppedAudioCheckpointSignal { + await stoppedAudioCheckpointSignal.wait() + } + cancelDictation(preserveStoppedAudio: true) } } @@ -1885,6 +2002,18 @@ class DictationSessionController: ObservableObject { } } + private func discardStoppedAudioRecovery( + transcriptPersisted: Bool = false, + explicitDiscard: Bool = false + ) { + guard DictationStoppedAudioRecoveryStore.cleanup( + stoppedAudioRecovery, + transcriptPersisted: transcriptPersisted, + explicitDiscard: explicitDiscard + ) else { return } + stoppedAudioRecovery = nil + } + private func startPersistingDictationTranscript( text: String, delivery: DictationDelivery diff --git a/Tests/DictationStoppedAudioRecoveryTests.swift b/Tests/DictationStoppedAudioRecoveryTests.swift new file mode 100644 index 000000000..b1c1ac634 --- /dev/null +++ b/Tests/DictationStoppedAudioRecoveryTests.swift @@ -0,0 +1,265 @@ +import Foundation + +func testDictationStoppedAudioRecovery() { + runSuite("Dictation stopped audio recovery writes a valid private WAV") { + let directory = makeRecoveryTestDirectory("wav") + defer { try? FileManager.default.removeItem(at: directory) } + let sessionID = UUID(uuidString: "00000000-0000-0000-0000-000000000123")! + + do { + let recovery = try DictationStoppedAudioRecoveryStore.persist( + samples16k: [-1, -0.5, 0, 0.5, 1], + sessionID: sessionID, + directory: directory + ) + assertNotNil(recovery, "non-empty stopped audio should be checkpointed") + guard let recovery else { return } + let data = try Data(contentsOf: recovery.url) + assertEqual(String(data: data[0..<4], encoding: .ascii), "RIFF", "recovery audio should use a WAV container") + assertEqual(String(data: data[8..<12], encoding: .ascii), "WAVE", "recovery audio should identify the WAV format") + assertEqual(readUInt32LE(data, offset: 24), 16_000, "recovery audio should be stored at the inference sample rate") + assertEqual(readUInt16LE(data, offset: 34), 16, "recovery audio should use 16-bit PCM") + assertEqual(readUInt32LE(data, offset: 40), 10, "WAV data length should match the sample count") + let attributes = try FileManager.default.attributesOfItem(atPath: recovery.url.path) + let permissions = (attributes[.posixPermissions] as? NSNumber)?.intValue + assertEqual(permissions, 0o600, "recovery audio should be owner-only") + let discovered = DictationStoppedAudioRecoveryStore.pendingRecoveries(limit: 1, directory: directory) + assertEqual(discovered, [recovery], "durable metadata should make recovery discoverable after store recreation") + } catch { + assertTrue(false, "recovery WAV should persist: \(error)") + } + } + + runSuite("Dictation stopped audio recovery survives failed transcript persistence") { + let directory = makeRecoveryTestDirectory("retain") + defer { try? FileManager.default.removeItem(at: directory) } + do { + let recovery = try DictationStoppedAudioRecoveryStore.persist( + samples16k: [0.25, -0.25], + sessionID: UUID(), + directory: directory + ) + let cleaned = DictationStoppedAudioRecoveryStore.cleanup( + recovery, + transcriptPersisted: false, + explicitDiscard: false + ) + assertFalse(cleaned, "failed transcript persistence must not clean recovery audio") + assertTrue(FileManager.default.fileExists(atPath: recovery!.url.path), "recovery audio must remain durable") + } catch { + assertTrue(false, "recovery audio should persist: \(error)") + } + } + + runSuite("Dictation stopped audio recovery cleans up only after success or explicit discard") { + for transcriptPersisted in [true, false] { + let directory = makeRecoveryTestDirectory(transcriptPersisted ? "saved" : "discarded") + defer { try? FileManager.default.removeItem(at: directory) } + do { + let recovery = try DictationStoppedAudioRecoveryStore.persist( + samples16k: [0.1], + sessionID: UUID(), + directory: directory + ) + let cleaned = DictationStoppedAudioRecoveryStore.cleanup( + recovery, + transcriptPersisted: transcriptPersisted, + explicitDiscard: !transcriptPersisted + ) + assertTrue(cleaned, "successful save or explicit discard should clean recovery audio") + assertFalse(FileManager.default.fileExists(atPath: recovery!.url.path), "cleaned recovery audio should be deleted") + assertTrue( + DictationStoppedAudioRecoveryStore.pendingRecoveries(directory: directory).isEmpty, + "cleanup should remove restart-discovery metadata" + ) + } catch { + assertTrue(false, "recovery cleanup should succeed: \(error)") + } + } + } + + runSuite("Dictation stopped audio recovery limits after newest-first ordering") { + let oldSessionID = UUID(uuidString: "00000000-0000-0000-0000-000000000001")! + let middleSessionID = UUID(uuidString: "00000000-0000-0000-0000-000000000002")! + let newSessionID = UUID(uuidString: "00000000-0000-0000-0000-000000000003")! + let recoveries = [ + DictationStoppedAudioRecovery(url: URL(fileURLWithPath: "/old.wav"), sessionID: oldSessionID, createdAt: Date(timeIntervalSince1970: 1)), + DictationStoppedAudioRecovery(url: URL(fileURLWithPath: "/new.wav"), sessionID: newSessionID, createdAt: Date(timeIntervalSince1970: 3)), + DictationStoppedAudioRecovery(url: URL(fileURLWithPath: "/middle.wav"), sessionID: middleSessionID, createdAt: Date(timeIntervalSince1970: 2)) + ] + + let limited = DictationStoppedAudioRecoveryStore.mostRecent(recoveries, limit: 2) + + assertEqual( + limited.map(\.sessionID), + [newSessionID, middleSessionID], + "the limit must select the newest recoveries regardless of enumeration order" + ) + } + + runSuite("Stopped audio persistence rejects cancelled and superseded sessions") { + let activeSessionID = UUID() + assertTrue( + DictationStoppedAudioRecoveryCommitPolicy.shouldPersist( + taskCancelled: false, + isDictating: true, + taskSessionID: activeSessionID, + currentSessionID: activeSessionID + ), + "the current live stop task should persist its recovery checkpoint" + ) + assertFalse( + DictationStoppedAudioRecoveryCommitPolicy.shouldPersist( + taskCancelled: true, + isDictating: true, + taskSessionID: activeSessionID, + currentSessionID: activeSessionID + ), + "a cancelled stop task must not persist after detached resampling returns" + ) + assertFalse( + DictationStoppedAudioRecoveryCommitPolicy.shouldPersist( + taskCancelled: false, + isDictating: true, + taskSessionID: activeSessionID, + currentSessionID: UUID() + ), + "an old stop task must not mutate a successor session" + ) + + assertTrue( + DictationStoppedAudioRecoveryCommitPolicy.shouldRetainPersistedRecovery( + taskSessionID: activeSessionID, + preservationSessionID: activeSessionID + ), + "termination cancellation must retain a checkpoint already written for that session" + ) + assertFalse( + DictationStoppedAudioRecoveryCommitPolicy.shouldRetainPersistedRecovery( + taskSessionID: activeSessionID, + preservationSessionID: UUID() + ), + "a successor session must not retain an old task's checkpoint" + ) + } + + runSuite("Dictation controller checkpoints audio before waiting for the model") { + do { + let source = try String( + contentsOf: repoFixtureURL("Sources/UI/Overlay/DictationSessionController.swift"), + encoding: .utf8 + ) + guard let persistRange = source.range(of: "DictationStoppedAudioRecoveryStore.persist("), + let modelWaitRange = source.range(of: "if !appState.sttRouter.isModelLoaded", range: persistRange.upperBound.. URL { + FileManager.default.temporaryDirectory + .appendingPathComponent("DictationStoppedAudioRecoveryTests-\(suffix)-\(UUID().uuidString)", isDirectory: true) +} + +private func readUInt16LE(_ data: Data, offset: Int) -> UInt16 { + UInt16(data[offset]) | (UInt16(data[offset + 1]) << 8) +} + +private func readUInt32LE(_ data: Data, offset: Int) -> UInt32 { + UInt32(data[offset]) + | (UInt32(data[offset + 1]) << 8) + | (UInt32(data[offset + 2]) << 16) + | (UInt32(data[offset + 3]) << 24) +} diff --git a/scripts/entrypoints/run-tests.sh b/scripts/entrypoints/run-tests.sh index 7cb0eeecf..01758d7af 100755 --- a/scripts/entrypoints/run-tests.sh +++ b/scripts/entrypoints/run-tests.sh @@ -289,6 +289,7 @@ APP_SOURCES=( "Sources/Support/ClipboardRestoringTextPaster.swift" "Sources/Accessibility/AccessibilityBridge.swift" "Sources/Dictation/DictationSessionTimeout.swift" + "Sources/Dictation/DictationStoppedAudioRecovery.swift" "Sources/Dictation/DictationStoragePaths.swift" "Sources/Dictation/DictationStopFinalizationPolicy.swift" "Sources/Dictation/DictationTranscriptWriter.swift"