From 6abdee4ed4f4f5b518784355d306ff6dd60e0294 Mon Sep 17 00:00:00 2001 From: Joshua Horton Date: Tue, 5 May 2026 16:06:00 -0500 Subject: [PATCH 1/3] fix(web): block use of predictable "fallback" result to cases where no corrections could be found Build-bot: skip build:web Test-bot: skip --- .../main/correction/tokenization-corrector.ts | 47 +++++++++----- .../tokenization-corrector.tests.ts | 65 +++++++++++++++++++ 2 files changed, 97 insertions(+), 15 deletions(-) diff --git a/web/src/engine/predictive-text/worker-thread/src/main/correction/tokenization-corrector.ts b/web/src/engine/predictive-text/worker-thread/src/main/correction/tokenization-corrector.ts index 1818fa51d9e..e47c6b04fad 100644 --- a/web/src/engine/predictive-text/worker-thread/src/main/correction/tokenization-corrector.ts +++ b/web/src/engine/predictive-text/worker-thread/src/main/correction/tokenization-corrector.ts @@ -62,6 +62,7 @@ export class TokenizationCorrector implements CorrectionSearchable; private lastTotalCost: number; private handleHasBeenCalled: boolean = false; + private predictableMatchFound: boolean = false; get currentCost(): number { const correctable = this.selectionQueue.peek(); @@ -288,16 +289,19 @@ export class TokenizationCorrector implements CorrectionSearchable correction-string map with the obtained result. this._generatedTokenResults.set(correctableToUpdate.spaceId, tokenResult.mapping); } @@ -351,11 +359,20 @@ export class TokenizationCorrector implements CorrectionSearchable { assert.equal(searchResult.type, 'none'); }); + it('finds a default correction for a single correctable token without a model match', () => { + const fixture = buildFixture_therefore(); + + const theref = fixture.theref.tail; + const xInput: ProbabilityMass = { + sample: { + insert: 'x', + deleteLeft: 0, + id: 123 + }, + p: 1 + } + const therefx = new SubstitutionQuotientSpur(theref.searchModule, [xInput], xInput); + const yInput: ProbabilityMass = { + sample: { + insert: 'y', + deleteLeft: 0, + id: 124 + }, + p: 1 + } + const therefxy = new SubstitutionQuotientSpur(therefx, [yInput], yInput); + const zInput: ProbabilityMass = { + sample: { + insert: 'z', + deleteLeft: 0, + id: 125 + }, + p: 1 + } + const therefxyz = new ContextToken(new SubstitutionQuotientSpur(therefxy, [zInput], zInput)); + const therefxyzTokenization = new ContextTokenization([therefxyz], null, null); + + const instance = new TokenizationCorrector( + therefxyzTokenization, + 1, + fixture.filter + ); + + let searchResult: PathResult; + do { + searchResult = instance.handleNextNode(); + } while(searchResult.type == 'intermediate'); + + assert.equal(searchResult.type, 'complete'); + if(searchResult.type == 'complete') { + const mapping = searchResult.mapping; + const tokenResults = mapping.matchedResult; + assert.isNotNaN(searchResult.cost); + assert.equal(searchResult.cost, searchResult.mapping.totalCost); + assert.equal(tokenResults.length, 1); + assert.sameOrderedMembers(tokenResults.map((r) => r.matchString), ['therefxyz']); + + // Now that an entry has been found, verify the corrector's state. + assert.isNotOk(instance.predictableToken); // should become an uncorrectable. + assert.isTrue(instance.generatedTokenResults.has(therefxyz)); + assert.equal(instance.generatedTokenResults.get(therefxyz), tokenResults[0]); + } + + // There should be no further possible suggestions. + searchResult = instance.handleNextNode(); + assert.equal(searchResult.type, 'none'); + }); + it('finds corrections for a group of tokens with two correctable', () => { const fixture = buildFixture_therefore(); From 6ca785cd13b1c0742d9c5048e3c6731da89d550f Mon Sep 17 00:00:00 2001 From: Joshua Horton Date: Fri, 8 May 2026 15:04:36 -0500 Subject: [PATCH 2/3] change(web): address PR suggestion of local var reuse --- .../src/main/correction/tokenization-corrector.ts | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/web/src/engine/predictive-text/worker-thread/src/main/correction/tokenization-corrector.ts b/web/src/engine/predictive-text/worker-thread/src/main/correction/tokenization-corrector.ts index e47c6b04fad..720db56ac86 100644 --- a/web/src/engine/predictive-text/worker-thread/src/main/correction/tokenization-corrector.ts +++ b/web/src/engine/predictive-text/worker-thread/src/main/correction/tokenization-corrector.ts @@ -277,8 +277,9 @@ export class TokenizationCorrector implements CorrectionSearchable { - if(correctableToUpdate != this._predictable) { + if(correctionIsThePredictable) { // Lock the 'correctable' token now that either a valid correction for // it has been found or all possible corrections are exhausted. We only // consider a single correction for most of a tokenization's tokens, @@ -289,8 +290,9 @@ export class TokenizationCorrector implements CorrectionSearchable Date: Thu, 14 May 2026 22:48:44 +0700 Subject: [PATCH 3/3] fix(web): apply logic fix from suggestion Co-authored-by: Eberhard Beilharz --- .../worker-thread/src/main/correction/tokenization-corrector.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/web/src/engine/predictive-text/worker-thread/src/main/correction/tokenization-corrector.ts b/web/src/engine/predictive-text/worker-thread/src/main/correction/tokenization-corrector.ts index 720db56ac86..a8b229f67b8 100644 --- a/web/src/engine/predictive-text/worker-thread/src/main/correction/tokenization-corrector.ts +++ b/web/src/engine/predictive-text/worker-thread/src/main/correction/tokenization-corrector.ts @@ -279,7 +279,7 @@ export class TokenizationCorrector implements CorrectionSearchable { - if(correctionIsThePredictable) { + if(!correctionIsThePredictable) { // Lock the 'correctable' token now that either a valid correction for // it has been found or all possible corrections are exhausted. We only // consider a single correction for most of a tokenization's tokens,