From 7d2e39d0a456749bc3c434e6d6dd38604362416b Mon Sep 17 00:00:00 2001 From: Alex-Wengg Date: Thu, 9 Jul 2026 19:59:19 -0400 Subject: [PATCH] fix(asr/streaming): temporally gate token dedup to stop prefix-collision loss MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit removeDuplicateTokenSequence matched purely on token-ID equality, so two different words spoken seconds apart that share a short subword prefix (e.g. Russian "трансформацию" vs "транскрибируется", both starting " тра"+"ран") were treated as a chunk-boundary duplicate. The shared prefix — including the leading-space word-start token — was stripped from accumulatedTokens, corrupting both live updates and finish()'s reconstruction (issue #787). Pair each token with its global encoder-frame timestamp and require matched occurrences to fall within duplicateFrameTolerance (2.0s) of each other. A genuine boundary duplicate lands at ~identical global time in both overlapping windows; a coincidental prefix match is many seconds apart and is now rejected. Falls back to id-only matching when timestamps are absent, so offline callers (ChunkProcessor, DualDecodeArbitration) are unchanged. - ASRConstants: add duplicateFrameTolerance (25 frames = 2.0s) - removeDuplicateTokenSequence: optional previous/current timestamps + tolerance, folded into the SequenceMatcher element matcher - transcribeChunk: forward previous timestamps + globalFrameOffset - SlidingWindowAsrManager: accumulate global token timestamps 1:1 with tokens - tests: #787 regression coverage (far-apart kept, close deduped, tolerance edge) --- .../SlidingWindowAsrManager.swift | 20 ++++ .../TDT/AsrManager+TokenProcessing.swift | 56 ++++++++++-- .../TDT/AsrManager+Transcription.swift | 10 +- Sources/FluidAudio/Shared/ASRConstants.swift | 11 +++ .../TokenDeduplicationRegressionTests.swift | 91 +++++++++++++++++++ 5 files changed, 179 insertions(+), 9 deletions(-) diff --git a/Sources/FluidAudio/ASR/Parakeet/SlidingWindow/SlidingWindowAsrManager.swift b/Sources/FluidAudio/ASR/Parakeet/SlidingWindow/SlidingWindowAsrManager.swift index b341e6fa8..8d01dbeef 100644 --- a/Sources/FluidAudio/ASR/Parakeet/SlidingWindow/SlidingWindowAsrManager.swift +++ b/Sources/FluidAudio/ASR/Parakeet/SlidingWindow/SlidingWindowAsrManager.swift @@ -31,6 +31,10 @@ public actor SlidingWindowAsrManager { private var segmentIndex: Int = 0 private var lastProcessedFrame: Int = 0 private var accumulatedTokens: [Int] = [] + // Global encoder-frame timestamp for each accumulated token (1:1 with + // accumulatedTokens). Lets per-chunk dedup require temporal adjacency so a + // coincidental subword-prefix match between far-apart words isn't dropped (#787). + private var accumulatedTokenTimestamps: [Int] = [] // Raw sample buffer for sliding-window assembly (absolute indexing) private var sampleBuffer: [Float] = [] @@ -180,6 +184,7 @@ public actor SlidingWindowAsrManager { segmentIndex = 0 lastProcessedFrame = 0 accumulatedTokens.removeAll() + accumulatedTokenTimestamps.removeAll() failedWindowCount = 0 lastWindowError = nil @@ -314,6 +319,7 @@ public actor SlidingWindowAsrManager { segmentIndex = 0 lastProcessedFrame = 0 accumulatedTokens.removeAll() + accumulatedTokenTimestamps.removeAll() logger.info("SlidingWindowAsrManager reset for source: \(String(describing: self.audioSource))") } @@ -445,6 +451,8 @@ public actor SlidingWindowAsrManager { windowSamples, decoderState: &state, previousTokens: accumulatedTokens, + previousTokenTimestamps: accumulatedTokenTimestamps, + globalFrameOffset: windowStartSample / ASRConstants.samplesPerEncoderFrame, isLastChunk: isLastChunk, language: config.language ) @@ -477,6 +485,18 @@ public actor SlidingWindowAsrManager { // Update state only after all required async calls complete successfully accumulatedTokens.append(contentsOf: tokens) + // Keep global timestamps aligned 1:1 with accumulatedTokens for #787 dedup. + // `tokens`/`adjustedTimestamps` are already post-dedup and same length; guard + // against any mismatch so the arrays never drift out of alignment. + if adjustedTimestamps.count == tokens.count { + accumulatedTokenTimestamps.append(contentsOf: adjustedTimestamps) + } else { + accumulatedTokenTimestamps.append(contentsOf: adjustedTimestamps.prefix(tokens.count)) + if adjustedTimestamps.count < tokens.count { + accumulatedTokenTimestamps.append( + contentsOf: Array(repeating: -1, count: tokens.count - adjustedTimestamps.count)) + } + } lastProcessedFrame = max(lastProcessedFrame, adjustedTimestamps.max() ?? 0) segmentIndex += 1 processedChunks += 1 diff --git a/Sources/FluidAudio/ASR/Parakeet/SlidingWindow/TDT/AsrManager+TokenProcessing.swift b/Sources/FluidAudio/ASR/Parakeet/SlidingWindow/TDT/AsrManager+TokenProcessing.swift index 4a33e5883..6e5441e93 100644 --- a/Sources/FluidAudio/ASR/Parakeet/SlidingWindow/TDT/AsrManager+TokenProcessing.swift +++ b/Sources/FluidAudio/ASR/Parakeet/SlidingWindow/TDT/AsrManager+TokenProcessing.swift @@ -107,34 +107,74 @@ extension AsrManager { return timings } + /// A token paired with the global encoder-frame index at which it was emitted. + /// Used to make dedup matching temporally aware; `timestamp < 0` means "unknown". + private struct TimedToken { + let id: Int + let timestamp: Int + } + /// Remove duplicate token sequences at the start of the current list that overlap /// with the tail of the previous accumulated tokens. Returns deduplicated current tokens /// and the number of removed leading tokens so caller can drop aligned timestamps. + /// + /// When `previousTimestamps` and `currentTimestamps` (both in **global** encoder frames) + /// are supplied, a token match only counts as a genuine chunk-boundary duplicate if the + /// two occurrences fall within `frameTolerance` frames of each other — i.e. they describe + /// the same acoustic moment in the overlap region. This prevents false positives where two + /// unrelated words spoken seconds apart share a short subword prefix (issue #787). Without + /// timestamps the matcher falls back to pure token-ID equality (legacy behavior). + /// /// Ideally this is not needed. We need to make some more fixes to the TDT decoding logic, /// this should be a temporary workaround. nonisolated internal func removeDuplicateTokenSequence( - previous: [Int], current: [Int], maxOverlap: Int = 12 + previous: [Int], current: [Int], maxOverlap: Int = 12, + previousTimestamps: [Int]? = nil, + currentTimestamps: [Int]? = nil, + frameTolerance: Int = ASRConstants.duplicateFrameTolerance ) -> (deduped: [Int], removedCount: Int) { // Handle single punctuation token duplicates first (domain-specific) let punctuationTokens = ASRConstants.punctuationTokens var workingCurrent = current + var workingCurrentTimestamps = currentTimestamps var removedCount = 0 if !previous.isEmpty && !workingCurrent.isEmpty && previous.last == workingCurrent.first && punctuationTokens.contains(workingCurrent.first!) { workingCurrent = Array(workingCurrent.dropFirst()) + workingCurrentTimestamps = workingCurrentTimestamps.map { Array($0.dropFirst()) } removedCount += 1 } + // Pair tokens with their global frame timestamps so the matcher can reject + // token-ID coincidences that don't line up in time. When timestamps are absent + // (or misaligned) the sentinel -1 makes the matcher fall back to ID-only equality. + func timed(_ ids: [Int], _ timestamps: [Int]?) -> [TimedToken] { + if let timestamps, timestamps.count == ids.count { + return zip(ids, timestamps).map { TimedToken(id: $0, timestamp: $1) } + } + return ids.map { TimedToken(id: $0, timestamp: -1) } + } + + let previousTimed = timed(previous, previousTimestamps) + let currentTimed = timed(workingCurrent, workingCurrentTimestamps) + + // Token IDs must match, and — when both timestamps are known — the two + // occurrences must describe the same acoustic moment (within tolerance). + let temporalMatcher: (TimedToken, TimedToken) -> Bool = { a, b in + guard a.id == b.id else { return false } + guard a.timestamp >= 0 && b.timestamp >= 0 else { return true } + return abs(a.timestamp - b.timestamp) <= frameTolerance + } + // STAGE 2: Suffix-prefix overlap using extracted utility - let exactMatcher: (Int, Int) -> Bool = { $0 == $1 } if let match = SequenceMatcher.findSuffixPrefixMatch( - previous: previous, - current: workingCurrent, + previous: previousTimed, + current: currentTimed, maxOverlap: maxOverlap, - matcher: exactMatcher + matcher: temporalMatcher ) { logger.debug("Found exact suffix-prefix overlap of length \(match.length)") let finalRemoved = removedCount + match.length @@ -146,11 +186,11 @@ extension AsrManager { let maxSearchLength = min(15, previous.count) if let match = SequenceMatcher.findBoundedSubstringMatch( - previous: previous, - current: workingCurrent, + previous: previousTimed, + current: currentTimed, maxSearchLength: maxSearchLength, boundarySearchFrames: boundarySearchFrames, - matcher: exactMatcher + matcher: temporalMatcher ) { logger.debug( "Found duplicate sequence length=\(match.length) at currStart=\(match.rightStartIndex) (boundarySearch=\(boundarySearchFrames))" diff --git a/Sources/FluidAudio/ASR/Parakeet/SlidingWindow/TDT/AsrManager+Transcription.swift b/Sources/FluidAudio/ASR/Parakeet/SlidingWindow/TDT/AsrManager+Transcription.swift index 4a29727da..3028fe50e 100644 --- a/Sources/FluidAudio/ASR/Parakeet/SlidingWindow/TDT/AsrManager+Transcription.swift +++ b/Sources/FluidAudio/ASR/Parakeet/SlidingWindow/TDT/AsrManager+Transcription.swift @@ -58,6 +58,8 @@ extension AsrManager { _ chunkSamples: [Float], decoderState: inout TdtDecoderState, previousTokens: [Int] = [], + previousTokenTimestamps: [Int]? = nil, + globalFrameOffset: Int = 0, isLastChunk: Bool = false, language: Language? = nil ) async throws -> (tokens: [Int], timestamps: [Int], confidences: [Float], encoderSequenceLength: Int) { @@ -76,8 +78,14 @@ extension AsrManager { // Apply token deduplication if previous tokens are provided if !previousTokens.isEmpty && hypothesis.hasTokens { + // Convert this chunk's local frame timestamps into the same global frame + // space as `previousTokenTimestamps` so dedup can require temporal adjacency. + let currentGlobalTimestamps: [Int]? = + previousTokenTimestamps != nil ? hypothesis.timestamps.map { $0 + globalFrameOffset } : nil let (deduped, removedCount) = removeDuplicateTokenSequence( - previous: previousTokens, current: hypothesis.ySequence) + previous: previousTokens, current: hypothesis.ySequence, + previousTimestamps: previousTokenTimestamps, + currentTimestamps: currentGlobalTimestamps) let adjustedTimestamps = removedCount > 0 ? Array(hypothesis.timestamps.dropFirst(removedCount)) : hypothesis.timestamps let adjustedConfidences = diff --git a/Sources/FluidAudio/Shared/ASRConstants.swift b/Sources/FluidAudio/Shared/ASRConstants.swift index 789f4da69..48d7c47b5 100644 --- a/Sources/FluidAudio/Shared/ASRConstants.swift +++ b/Sources/FluidAudio/Shared/ASRConstants.swift @@ -48,6 +48,17 @@ public enum ASRConstants { /// Standard overlap in encoder frames (2.0s = 25 frames at 0.08s per frame) public static let standardOverlapFrames: Int = 25 + /// Maximum global-frame gap between two token occurrences for them to be + /// treated as the *same* acoustic event during sliding-window token dedup. + /// + /// A genuine chunk-boundary duplicate lands at nearly identical global audio + /// time in both overlapping windows (difference ~0, plus a few frames of + /// cross-window emission jitter). A coincidental subword-prefix match between + /// two *different* words spoken seconds apart is far outside this bound, so + /// gating on it prevents false-positive dedup that drops real tokens. + /// See issue #787. + public static let duplicateFrameTolerance: Int = standardOverlapFrames // 25 frames = 2.0s + /// Minimum confidence score (for empty or very uncertain transcriptions) public static let minConfidence: Float = 0.1 diff --git a/Tests/FluidAudioTests/ASR/TokenDeduplicationRegressionTests.swift b/Tests/FluidAudioTests/ASR/TokenDeduplicationRegressionTests.swift index 4f7a4cd75..1891d6eeb 100644 --- a/Tests/FluidAudioTests/ASR/TokenDeduplicationRegressionTests.swift +++ b/Tests/FluidAudioTests/ASR/TokenDeduplicationRegressionTests.swift @@ -138,6 +138,97 @@ final class TokenDeduplicationRegressionTests: XCTestCase { XCTAssertEqual(removed1, 1, "Should report 1 removed (punctuation)") } + // MARK: - Issue #787: Temporal gating of prefix-collision dedup + + /// Token IDs from issue #787: two different Russian words that share the + /// leading subword prefix " тра" + "ран" (= " тран"). + /// 6841 = " тра" (leading space / word-start marker), 2394 = "ран". + private static let sharedPrefixA = 6841 // " тра" + private static let sharedPrefixB = 2394 // "ран" + + /// The earlier word (трансформацию) sits in the tail of the accumulated + /// history; the later, different word (транскрибируется) opens the new chunk. + private var previousWithEarlierWord: [Int] { + [10, 11, 12, Self.sharedPrefixA, Self.sharedPrefixB, 9001, 13, 14, 15] + } + private var currentWithLaterWord: [Int] { + [Self.sharedPrefixA, Self.sharedPrefixB, 9002, 20, 21] + } + + /// Without timestamps the legacy id-only matcher still strips the shared + /// prefix — this documents the #787 bug and pins the fallback behavior. + func testDedup_787_NoTimestamps_LegacyStripsSharedPrefix() { + let asrManager = AsrManager() + let (deduped, removed) = asrManager.removeDuplicateTokenSequence( + previous: previousWithEarlierWord, + current: currentWithLaterWord + ) + XCTAssertEqual(deduped, [9002, 20, 21], "Legacy id-only path removes the shared prefix") + XCTAssertEqual(removed, 2) + } + + /// With global timestamps that put the two occurrences ~10s apart, the shared + /// prefix must NOT be treated as a duplicate — the later word stays intact. + func testDedup_787_FarApartTimestamps_KeepsSharedPrefix() { + let asrManager = AsrManager() + // 6841@frame 3, 2394@frame 4 in the earlier word. + let previousTs = [0, 1, 2, 3, 4, 5, 6, 7, 8] + // Same token IDs but ~10.4s (130 frames) later — a different utterance. + let currentTs = [130, 131, 132, 133, 134] + + let (deduped, removed) = asrManager.removeDuplicateTokenSequence( + previous: previousWithEarlierWord, + current: currentWithLaterWord, + previousTimestamps: previousTs, + currentTimestamps: currentTs + ) + XCTAssertEqual( + deduped, currentWithLaterWord, + "Temporally distant prefix collision must not be deduped (#787)") + XCTAssertEqual(removed, 0) + } + + /// A genuine chunk-boundary duplicate (same acoustic moment, near-identical + /// global timestamps) must still be deduped when timestamps are supplied. + func testDedup_787_CloseTimestamps_StillDedupes() { + let asrManager = AsrManager() + let previousTs = [0, 1, 2, 3, 4, 5, 6, 7, 8] + // Same region re-decoded in the overlapping window: a few frames of jitter. + let currentTs = [3, 4, 5, 6, 7] + + let (deduped, removed) = asrManager.removeDuplicateTokenSequence( + previous: previousWithEarlierWord, + current: currentWithLaterWord, + previousTimestamps: previousTs, + currentTimestamps: currentTs + ) + XCTAssertEqual(deduped, [9002, 20, 21], "Real boundary overlap should still be deduped") + XCTAssertEqual(removed, 2) + } + + /// A gap just outside `frameTolerance` is rejected; just inside is accepted. + func testDedup_787_ToleranceBoundary() { + let asrManager = AsrManager() + let previousTs = [0, 1, 2, 3, 4, 5, 6, 7, 8] + let tolerance = ASRConstants.duplicateFrameTolerance + + // prefix at prev frame 3; current just beyond tolerance -> keep + let justOutside = [3 + tolerance + 1, 3 + tolerance + 2, 100, 101, 102] + let (dedupedOut, removedOut) = asrManager.removeDuplicateTokenSequence( + previous: previousWithEarlierWord, current: currentWithLaterWord, + previousTimestamps: previousTs, currentTimestamps: justOutside) + XCTAssertEqual(removedOut, 0, "Gap beyond tolerance is not a duplicate") + XCTAssertEqual(dedupedOut, currentWithLaterWord) + + // current just within tolerance -> dedup + let justInside = [3 + tolerance, 4 + tolerance, 100, 101, 102] + let (dedupedIn, removedIn) = asrManager.removeDuplicateTokenSequence( + previous: previousWithEarlierWord, current: currentWithLaterWord, + previousTimestamps: previousTs, currentTimestamps: justInside) + XCTAssertEqual(removedIn, 2, "Gap within tolerance is a duplicate") + XCTAssertEqual(dedupedIn, [9002, 20, 21]) + } + // MARK: - SequenceMatcher Utility Tests /// Test SequenceMatcher.findSuffixPrefixMatch