diff --git a/web/src/engine/predictive-text/worker-thread/src/main/correction/tokenization-corrector.ts b/web/src/engine/predictive-text/worker-thread/src/main/correction/tokenization-corrector.ts index 1818fa51d9e..a8b229f67b8 100644 --- a/web/src/engine/predictive-text/worker-thread/src/main/correction/tokenization-corrector.ts +++ b/web/src/engine/predictive-text/worker-thread/src/main/correction/tokenization-corrector.ts @@ -62,6 +62,7 @@ export class TokenizationCorrector implements CorrectionSearchable; private lastTotalCost: number; private handleHasBeenCalled: boolean = false; + private predictableMatchFound: boolean = false; get currentCost(): number { const correctable = this.selectionQueue.peek(); @@ -276,8 +277,9 @@ export class TokenizationCorrector implements CorrectionSearchable { - if(correctableToUpdate != this._predictable) { + if(!correctionIsThePredictable) { // Lock the 'correctable' token now that either a valid correction for // it has been found or all possible corrections are exhausted. We only // consider a single correction for most of a tokenization's tokens, @@ -289,18 +291,24 @@ export class TokenizationCorrector implements CorrectionSearchable correction-string map with the obtained result. this._generatedTokenResults.set(correctableToUpdate.spaceId, tokenResult.mapping); } @@ -351,11 +363,20 @@ export class TokenizationCorrector implements CorrectionSearchable { assert.equal(searchResult.type, 'none'); }); + it('finds a default correction for a single correctable token without a model match', () => { + const fixture = buildFixture_therefore(); + + const theref = fixture.theref.tail; + const xInput: ProbabilityMass = { + sample: { + insert: 'x', + deleteLeft: 0, + id: 123 + }, + p: 1 + } + const therefx = new SubstitutionQuotientSpur(theref.searchModule, [xInput], xInput); + const yInput: ProbabilityMass = { + sample: { + insert: 'y', + deleteLeft: 0, + id: 124 + }, + p: 1 + } + const therefxy = new SubstitutionQuotientSpur(therefx, [yInput], yInput); + const zInput: ProbabilityMass = { + sample: { + insert: 'z', + deleteLeft: 0, + id: 125 + }, + p: 1 + } + const therefxyz = new ContextToken(new SubstitutionQuotientSpur(therefxy, [zInput], zInput)); + const therefxyzTokenization = new ContextTokenization([therefxyz], null, null); + + const instance = new TokenizationCorrector( + therefxyzTokenization, + 1, + fixture.filter + ); + + let searchResult: PathResult; + do { + searchResult = instance.handleNextNode(); + } while(searchResult.type == 'intermediate'); + + assert.equal(searchResult.type, 'complete'); + if(searchResult.type == 'complete') { + const mapping = searchResult.mapping; + const tokenResults = mapping.matchedResult; + assert.isNotNaN(searchResult.cost); + assert.equal(searchResult.cost, searchResult.mapping.totalCost); + assert.equal(tokenResults.length, 1); + assert.sameOrderedMembers(tokenResults.map((r) => r.matchString), ['therefxyz']); + + // Now that an entry has been found, verify the corrector's state. + assert.isNotOk(instance.predictableToken); // should become an uncorrectable. + assert.isTrue(instance.generatedTokenResults.has(therefxyz)); + assert.equal(instance.generatedTokenResults.get(therefxyz), tokenResults[0]); + } + + // There should be no further possible suggestions. + searchResult = instance.handleNextNode(); + assert.equal(searchResult.type, 'none'); + }); + it('finds corrections for a group of tokens with two correctable', () => { const fixture = buildFixture_therefore();