refactor(web): refactor intermediate composited prediction type 🚂

web/src/engine/predictive-text/worker-thread/src/main/correction/tokenization-corrector.ts

@@ -58,6 +58,7 @@ export class TokenizationCorrector implements CorrectionSearchable<ReadonlyArray
   private _previousResults: TokenizationResultMapping[] = [];
 
   // fully private
+  public readonly modelsCorrectables: boolean;
   private selectionQueue: PriorityQueue<QuotientNodeFinalizer>;
   private tokenCostMap: Map<number, number>;
   private tokenLookupMap: Map<number, ContextToken>;
@@ -172,13 +173,16 @@ export class TokenizationCorrector implements CorrectionSearchable<ReadonlyArray
     this._correctables = [];
 
     this.tokenLookupMap = new Map();
+    let modelsCorrectables = false;
 
     orderedTokens.forEach((token, index) => {
       // New issue:  this mangles the space IDs!  We almost certainly need some
       // sort of proper map to the source token.
       const searchModule = new QuotientNodeFinalizer(token.searchModule, index == orderedTokens.length - 1);
       this.tokenLookupMap.set(searchModule.spaceId, token);
-      if(!filterClosure(token)) {
+      const passesFilter = filterClosure(token);
+      modelsCorrectables ||= passesFilter;
+      if(!passesFilter) {
         this._uncorrectables.push(searchModule);
       } else if(index == tailCorrectionLength - 1) {
         // The sole assignment case for this field.  It may only be assigned for
@@ -189,6 +193,8 @@ export class TokenizationCorrector implements CorrectionSearchable<ReadonlyArray
         this._correctables.push(searchModule);
       }
     });
+    // Set a readonly flag indicating if this Corrector started with correctable entries.
+    this.modelsCorrectables = modelsCorrectables;
 
     this._generatedTokenResults = new Map();
     const uncorrectables = this._uncorrectables;

web/src/engine/predictive-text/worker-thread/src/main/model-compositor.ts

@@ -1,11 +1,11 @@
 import * as models from '@keymanapp/models-templates';
 import { LexicalModelTypes } from '@keymanapp/common-types';
 
-import * as correction from './correction/index.js'
-import TransformUtils from './transformUtils.js';
 import { applySuggestionCasing, correctAndEnumerate, createDefaultKeep, dedupeSuggestions, finalizeSuggestions, predictionAutoSelect, processSimilarity, toAnnotatedSuggestion, tupleDisplayOrderSort } from './predict-helpers.js';
 import { detectCurrentCasing, determineModelTokenizer, determineModelWordbreaker, determinePunctuationFromModel } from './model-helpers.js';
+import TransformUtils from './transformUtils.js';
 
+import * as correction from './correction/index.js'
 import { ContextTracker } from './correction/context-tracker.js';
 import { DEFAULT_ALLOTTED_CORRECTION_TIME_INTERVAL } from './correction/distance-modeler.js';
 
@@ -162,10 +162,13 @@ export class ModelCompositor {
     // lexicon for a word.  (Example:  "Apple" the company vs "apple" the fruit.)
     for(let tuple of rawPredictions) {
       if(currentCasing && currentCasing != 'lower') {
-        applySuggestionCasing(tuple.prediction.sample, basePrefix, this.lexicalModel, currentCasing);
+        applySuggestionCasing(tuple.components.prediction, basePrefix, this.lexicalModel, currentCasing);
       }
     }
 
+    // what if... we fuse suggestions together here, after the 'apply casing' step?
+    // deduplication, etc function fine from a fused-prediction perspective here.
+
     // We want to dedupe before trimming the list so that we can present a full set
     // of viable distinct suggestions if available.
     const deduplicatedSuggestionTuples = dedupeSuggestions(this.lexicalModel, rawPredictions, context);