diff --git a/src/lexer/Lexer.ts b/src/lexer/Lexer.ts index 6f45df185..0048df286 100644 --- a/src/lexer/Lexer.ts +++ b/src/lexer/Lexer.ts @@ -1,5 +1,5 @@ /* eslint-disable func-names */ -import { TokenKind, ReservedWords, Keywords, PreceedingRegexTypes } from './TokenKind'; +import { TokenKind, ReservedWords, Keywords, PreceedingRegexTypes, FixedTokenText, LexerTextCache } from './TokenKind'; import type { Token } from './Token'; import { isAlpha, isDecimalDigit, isAlphaNumeric, isHexDigit } from './Characters'; import type { Range, Diagnostic } from 'vscode-languageserver'; @@ -105,8 +105,8 @@ export class Lexer { this.tokens.push({ kind: TokenKind.Eof, - isReserved: false, text: '', + isReserved: false, range: this.options.trackLocations ? util.createRange(this.lineBegin, this.columnBegin, this.lineEnd, this.columnEnd + 1) : undefined, @@ -392,13 +392,27 @@ export class Lexer { while (this.peek() === ' ' || this.peek() === '\t') { this.advance(); } - const whitespaceToken = this.addToken(TokenKind.Whitespace); - this.leadingWhitespace = whitespaceToken.text; - //if we aren't keeping the whitespace tokens, then remove this one if (this.options.includeWhitespace === false) { - this.tokens.pop(); + //skip the Token + Range allocations entirely; we only need the canonical + //text so subsequent tokens can reference it as their leadingWhitespace + let text = this.source.slice(this.start, this.current); + const cached = LexerTextCache.get(text); + if (cached !== undefined) { + text = cached; + } else { + LexerTextCache.set(text, text); + } + this.leadingWhitespace = text; + //match what addToken's sync() would have done so the next token's range + //starts after the whitespace rather than where it began + this.start = this.current; + this.lineBegin = this.lineEnd; + this.columnBegin = this.columnEnd; + } else { + const whitespaceToken = this.addToken(TokenKind.Whitespace); + this.leadingWhitespace = whitespaceToken.text; + this.start = this.current; } - this.start = this.current; } private newline() { @@ -1055,7 +1069,23 @@ export class Lexer { * @param kind the type of token to produce. */ private addToken(kind: TokenKind) { - let text = this.source.slice(this.start, this.current); + let text: string; + const fixedText = FixedTokenText[kind]; + if (fixedText !== undefined) { + text = fixedText; + } else { + text = this.source.slice(this.start, this.current); + //canonicalize tokens with a bounded set of valid text values so repeated + //occurrences share a single string instance instead of fresh sliced wrappers + if (kind === TokenKind.Newline || kind === TokenKind.Whitespace) { + const cached = LexerTextCache.get(text); + if (cached !== undefined) { + text = cached; + } else { + LexerTextCache.set(text, text); + } + } + } let token: Token = { kind: kind, text: text, diff --git a/src/lexer/TokenKind.ts b/src/lexer/TokenKind.ts index 10eb915ca..e97673251 100644 --- a/src/lexer/TokenKind.ts +++ b/src/lexer/TokenKind.ts @@ -704,3 +704,70 @@ export const PreceedingRegexTypes = new Set([ TokenKind.Colon, TokenKind.Semicolon ]); + +/** + * Token kinds whose source text is invariant (always the same character sequence). + * For these kinds the lexer can use the canonical string here instead of allocating + * a fresh substring/slice per token, which dramatically reduces per-token wrapper + * allocations across an entire build. Excludes any kind whose text could legitimately + * vary by case (keywords like `Function`/`function`) or content (identifiers, literals, + * comments, whitespace, multi-form sequences like Newline). + */ +export const FixedTokenText: Partial> = { + [TokenKind.LeftParen]: '(', + [TokenKind.RightParen]: ')', + [TokenKind.LeftSquareBracket]: '[', + [TokenKind.RightSquareBracket]: ']', + [TokenKind.LeftCurlyBrace]: '{', + [TokenKind.RightCurlyBrace]: '}', + [TokenKind.Caret]: '^', + [TokenKind.Minus]: '-', + [TokenKind.Plus]: '+', + [TokenKind.Star]: '*', + [TokenKind.Forwardslash]: '/', + [TokenKind.Backslash]: '\\', + [TokenKind.PlusPlus]: '++', + [TokenKind.MinusMinus]: '--', + [TokenKind.LeftShift]: '<<', + [TokenKind.RightShift]: '>>', + [TokenKind.MinusEqual]: '-=', + [TokenKind.PlusEqual]: '+=', + [TokenKind.StarEqual]: '*=', + [TokenKind.ForwardslashEqual]: '/=', + [TokenKind.BackslashEqual]: '\\=', + [TokenKind.LeftShiftEqual]: '<<=', + [TokenKind.RightShiftEqual]: '>>=', + [TokenKind.Less]: '<', + [TokenKind.LessEqual]: '<=', + [TokenKind.Greater]: '>', + [TokenKind.GreaterEqual]: '>=', + [TokenKind.Equal]: '=', + [TokenKind.LessGreater]: '<>', + [TokenKind.Dot]: '.', + [TokenKind.Comma]: ',', + [TokenKind.Colon]: ':', + [TokenKind.Semicolon]: ';', + [TokenKind.At]: '@', + [TokenKind.Callfunc]: '@.', + [TokenKind.Question]: '?', + [TokenKind.QuestionQuestion]: '??', + [TokenKind.BackTick]: '`', + [TokenKind.QuestionDot]: '?.', + [TokenKind.QuestionLeftSquare]: '?[', + [TokenKind.QuestionLeftParen]: '?(', + [TokenKind.QuestionAt]: '?@', + [TokenKind.Dollar]: '$', + [TokenKind.Eof]: '' +}; + +/** + * Lazy intern table for `Newline` and `Whitespace` token text. Real source uses + * a small number of unique values for each (3 valid newline forms, ~50 typical + * indent patterns), so canonicalizing on first sight collapses per-token sliced + * string wrappers without the per-call overhead that a full token-text interner + * would incur on the unbounded `Identifier` and literal token kinds. + * + * Module-scope retention is trivial (~a few KB) and the Map grows only on + * first-seen text within a process lifetime. + */ +export const LexerTextCache = new Map(); diff --git a/src/parser/Parser.ts b/src/parser/Parser.ts index 969f34bd0..c1dfc062f 100644 --- a/src/parser/Parser.ts +++ b/src/parser/Parser.ts @@ -859,9 +859,9 @@ export class Parser { range: this.peek().range }); functionType = { - isReserved: true, kind: TokenKind.Function, text: 'function', + isReserved: true, //zero-length location means derived range: { start: this.peek().range.start, @@ -1092,7 +1092,7 @@ export class Parser { } else { const nameExpression = new VariableExpression(name); result = new AssignmentStatement( - { kind: TokenKind.Equal, text: '=', range: operator.range }, + { kind: TokenKind.Equal, text: '=', isReserved: false, range: operator.range, leadingWhitespace: '' }, name, new BinaryExpression(nameExpression, operator, value) ); @@ -2287,7 +2287,7 @@ export class Parser { left.additionalIndexes, operator.kind === TokenKind.Equal ? operator - : { kind: TokenKind.Equal, text: '=', range: operator.range } + : { kind: TokenKind.Equal, text: '=', isReserved: false, range: operator.range, leadingWhitespace: '' } ); } else if (isDottedGetExpression(left)) { return new DottedSetStatement( @@ -2299,7 +2299,7 @@ export class Parser { left.dot, operator.kind === TokenKind.Equal ? operator - : { kind: TokenKind.Equal, text: '=', range: operator.range } + : { kind: TokenKind.Equal, text: '=', isReserved: false, range: operator.range, leadingWhitespace: '' } ); } }