Reduce per-Token allocation in the lexer

src/lexer/Lexer.ts

@@ -1,5 +1,5 @@
 /* eslint-disable func-names */
-import { TokenKind, ReservedWords, Keywords, PreceedingRegexTypes } from './TokenKind';
+import { TokenKind, ReservedWords, Keywords, PreceedingRegexTypes, FixedTokenText, LexerTextCache } from './TokenKind';
 import type { Token } from './Token';
 import { isAlpha, isDecimalDigit, isAlphaNumeric, isHexDigit } from './Characters';
 import type { Range, Diagnostic } from 'vscode-languageserver';
@@ -105,8 +105,8 @@ export class Lexer {
 
         this.tokens.push({
             kind: TokenKind.Eof,
-            isReserved: false,
             text: '',
+            isReserved: false,
             range: this.options.trackLocations
                 ? util.createRange(this.lineBegin, this.columnBegin, this.lineEnd, this.columnEnd + 1)
                 : undefined,
@@ -392,13 +392,27 @@ export class Lexer {
         while (this.peek() === ' ' || this.peek() === '\t') {
             this.advance();
         }
-        const whitespaceToken = this.addToken(TokenKind.Whitespace);
-        this.leadingWhitespace = whitespaceToken.text;
-        //if we aren't keeping the whitespace tokens, then remove this one
         if (this.options.includeWhitespace === false) {
-            this.tokens.pop();
+            //skip the Token + Range allocations entirely; we only need the canonical
+            //text so subsequent tokens can reference it as their leadingWhitespace
+            let text = this.source.slice(this.start, this.current);
+            const cached = LexerTextCache.get(text);
+            if (cached !== undefined) {
+                text = cached;
+            } else {
+                LexerTextCache.set(text, text);
+            }
+            this.leadingWhitespace = text;
+            //match what addToken's sync() would have done so the next token's range
+            //starts after the whitespace rather than where it began
+            this.start = this.current;
+            this.lineBegin = this.lineEnd;
+            this.columnBegin = this.columnEnd;
+        } else {
+            const whitespaceToken = this.addToken(TokenKind.Whitespace);
+            this.leadingWhitespace = whitespaceToken.text;
+            this.start = this.current;
         }
-        this.start = this.current;
     }
 
     private newline() {
@@ -1055,7 +1069,23 @@ export class Lexer {
      * @param kind the type of token to produce.
      */
     private addToken(kind: TokenKind) {
-        let text = this.source.slice(this.start, this.current);
+        let text: string;
+        const fixedText = FixedTokenText[kind];
+        if (fixedText !== undefined) {
+            text = fixedText;
+        } else {
+            text = this.source.slice(this.start, this.current);
+            //canonicalize tokens with a bounded set of valid text values so repeated
+            //occurrences share a single string instance instead of fresh sliced wrappers
+            if (kind === TokenKind.Newline || kind === TokenKind.Whitespace) {
+                const cached = LexerTextCache.get(text);
+                if (cached !== undefined) {
+                    text = cached;
+                } else {
+                    LexerTextCache.set(text, text);
+                }
+            }
+        }
         let token: Token = {
             kind: kind,
             text: text,

src/lexer/TokenKind.ts

@@ -704,3 +704,70 @@ export const PreceedingRegexTypes = new Set([
     TokenKind.Colon,
     TokenKind.Semicolon
 ]);
+
+/**
+ * Token kinds whose source text is invariant (always the same character sequence).
+ * For these kinds the lexer can use the canonical string here instead of allocating
+ * a fresh substring/slice per token, which dramatically reduces per-token wrapper
+ * allocations across an entire build. Excludes any kind whose text could legitimately
+ * vary by case (keywords like `Function`/`function`) or content (identifiers, literals,
+ * comments, whitespace, multi-form sequences like Newline).
+ */
+export const FixedTokenText: Partial<Record<TokenKind, string>> = {
+    [TokenKind.LeftParen]: '(',
+    [TokenKind.RightParen]: ')',
+    [TokenKind.LeftSquareBracket]: '[',
+    [TokenKind.RightSquareBracket]: ']',
+    [TokenKind.LeftCurlyBrace]: '{',
+    [TokenKind.RightCurlyBrace]: '}',
+    [TokenKind.Caret]: '^',
+    [TokenKind.Minus]: '-',
+    [TokenKind.Plus]: '+',
+    [TokenKind.Star]: '*',
+    [TokenKind.Forwardslash]: '/',
+    [TokenKind.Backslash]: '\\',
+    [TokenKind.PlusPlus]: '++',
+    [TokenKind.MinusMinus]: '--',
+    [TokenKind.LeftShift]: '<<',
+    [TokenKind.RightShift]: '>>',
+    [TokenKind.MinusEqual]: '-=',
+    [TokenKind.PlusEqual]: '+=',
+    [TokenKind.StarEqual]: '*=',
+    [TokenKind.ForwardslashEqual]: '/=',
+    [TokenKind.BackslashEqual]: '\\=',
+    [TokenKind.LeftShiftEqual]: '<<=',
+    [TokenKind.RightShiftEqual]: '>>=',
+    [TokenKind.Less]: '<',
+    [TokenKind.LessEqual]: '<=',
+    [TokenKind.Greater]: '>',
+    [TokenKind.GreaterEqual]: '>=',
+    [TokenKind.Equal]: '=',
+    [TokenKind.LessGreater]: '<>',
+    [TokenKind.Dot]: '.',
+    [TokenKind.Comma]: ',',
+    [TokenKind.Colon]: ':',
+    [TokenKind.Semicolon]: ';',
+    [TokenKind.At]: '@',
+    [TokenKind.Callfunc]: '@.',
+    [TokenKind.Question]: '?',
+    [TokenKind.QuestionQuestion]: '??',
+    [TokenKind.BackTick]: '`',
+    [TokenKind.QuestionDot]: '?.',
+    [TokenKind.QuestionLeftSquare]: '?[',
+    [TokenKind.QuestionLeftParen]: '?(',
+    [TokenKind.QuestionAt]: '?@',
+    [TokenKind.Dollar]: '$',
+    [TokenKind.Eof]: ''
+};
+
+/**
+ * Lazy intern table for `Newline` and `Whitespace` token text. Real source uses
+ * a small number of unique values for each (3 valid newline forms, ~50 typical
+ * indent patterns), so canonicalizing on first sight collapses per-token sliced
+ * string wrappers without the per-call overhead that a full token-text interner
+ * would incur on the unbounded `Identifier` and literal token kinds.
+ *
+ * Module-scope retention is trivial (~a few KB) and the Map grows only on
+ * first-seen text within a process lifetime.
+ */
+export const LexerTextCache = new Map<string, string>();

src/parser/Parser.ts

@@ -859,9 +859,9 @@ export class Parser {
                     range: this.peek().range
                 });
                 functionType = {
-                    isReserved: true,
                     kind: TokenKind.Function,
                     text: 'function',
+                    isReserved: true,
                     //zero-length location means derived
                     range: {
                         start: this.peek().range.start,
@@ -1092,7 +1092,7 @@ export class Parser {
         } else {
             const nameExpression = new VariableExpression(name);
             result = new AssignmentStatement(
-                { kind: TokenKind.Equal, text: '=', range: operator.range },
+                { kind: TokenKind.Equal, text: '=', isReserved: false, range: operator.range, leadingWhitespace: '' },
                 name,
                 new BinaryExpression(nameExpression, operator, value)
             );
@@ -2287,7 +2287,7 @@ export class Parser {
                     left.additionalIndexes,
                     operator.kind === TokenKind.Equal
                         ? operator
-                        : { kind: TokenKind.Equal, text: '=', range: operator.range }
+                        : { kind: TokenKind.Equal, text: '=', isReserved: false, range: operator.range, leadingWhitespace: '' }
                 );
             } else if (isDottedGetExpression(left)) {
                 return new DottedSetStatement(
@@ -2299,7 +2299,7 @@ export class Parser {
                     left.dot,
                     operator.kind === TokenKind.Equal
                         ? operator
-                        : { kind: TokenKind.Equal, text: '=', range: operator.range }
+                        : { kind: TokenKind.Equal, text: '=', isReserved: false, range: operator.range, leadingWhitespace: '' }
                 );
             }
         }