From 30a15de84e575174e240491b1ebcee7d8825e8de Mon Sep 17 00:00:00 2001 From: Robin Linden Date: Thu, 20 Aug 2026 21:59:59 +0200 Subject: [PATCH] starlark: Implement indentation tokenization --- starlark/parser.h | 35 ++++++------ starlark/parser_test.cc | 5 +- starlark/token.h | 14 +++++ starlark/tokenizer.h | 58 ++++++++++++++++++++ starlark/tokenizer_test.cc | 109 +++++++++++++++++++++++++++++++++++++ 5 files changed, 204 insertions(+), 17 deletions(-) diff --git a/starlark/parser.h b/starlark/parser.h index d8b1005..c015349 100644 --- a/starlark/parser.h +++ b/starlark/parser.h @@ -114,14 +114,17 @@ class Parser { return tokenizer_.tokenize(); } - std::optional next_non_newline_token() { + // Starlark ignores things like newlines and indents in some contexts. + std::optional next_interesting_token() { while (true) { auto token = next_token(); if (!token) { return std::nullopt; } - if (!std::holds_alternative(*token)) { + if (!std::holds_alternative(*token) && + !std::holds_alternative(*token) && + !std::holds_alternative(*token)) { return token; } } @@ -145,7 +148,7 @@ class Parser { if (std::holds_alternative(token)) { std::vector elements; while (true) { - auto maybe_token = next_non_newline_token(); + auto maybe_token = next_interesting_token(); if (!maybe_token) { std::cerr << "Tokenization error in list expression.\n"; return std::nullopt; @@ -164,22 +167,22 @@ class Parser { } // On the first iteration, we check if this is a list comprehension. - auto maybe_next = next_non_newline_token(); + auto maybe_next = next_interesting_token(); if (elements.empty() && maybe_next.has_value() && std::holds_alternative(*maybe_next)) { - auto var_token = next_non_newline_token(); + auto var_token = next_interesting_token(); if (!var_token || !std::holds_alternative(*var_token)) { std::cerr << "Expected identifier in list comprehension.\n"; return std::nullopt; } - auto in_token = next_non_newline_token(); + auto in_token = next_interesting_token(); if (!in_token || !std::holds_alternative(*in_token)) { std::cerr << "Expected 'in' in list comprehension.\n"; return std::nullopt; } - auto iterable_token = next_non_newline_token(); + auto iterable_token = next_interesting_token(); if (!iterable_token) { std::cerr << "Unexpected end of input in list comprehension.\n"; return std::nullopt; @@ -191,7 +194,7 @@ class Parser { return std::nullopt; } - auto maybe_closing = next_non_newline_token(); + auto maybe_closing = next_interesting_token(); if (!maybe_closing || !std::holds_alternative(*maybe_closing)) { std::cerr << "Expected closing ']' in list comprehension.\n"; @@ -236,7 +239,7 @@ class Parser { std::vector> entries; while (true) { - auto maybe_token = next_non_newline_token(); + auto maybe_token = next_interesting_token(); if (!maybe_token) { std::cerr << "Tokenization error in dict expression.\n"; return std::nullopt; @@ -254,13 +257,13 @@ class Parser { return std::nullopt; } - auto colon_token = next_non_newline_token(); + auto colon_token = next_interesting_token(); if (!colon_token || !std::holds_alternative(*colon_token)) { std::cerr << "Expected ':' after dict key.\n"; return std::nullopt; } - auto value_token = next_non_newline_token(); + auto value_token = next_interesting_token(); if (!value_token) { std::cerr << "Unexpected end of input in dict expression.\n"; return std::nullopt; @@ -274,7 +277,7 @@ class Parser { entries.emplace_back(std::move(*key_expr), std::move(*value_expr)); - auto maybe_next = next_non_newline_token(); + auto maybe_next = next_interesting_token(); if (!maybe_next) { std::cerr << "Tokenization error in dict expression.\n"; return std::nullopt; @@ -380,7 +383,7 @@ class Parser { bool seen_kw_arg = false; while (true) { - auto maybe_token = next_non_newline_token(); + auto maybe_token = next_interesting_token(); if (!maybe_token) { std::cerr << "Unexpected end of input in argument list.\n"; return std::nullopt; @@ -396,7 +399,7 @@ class Parser { } if (std::holds_alternative(token)) { - maybe_token = next_non_newline_token(); + maybe_token = next_interesting_token(); if (!maybe_token) { std::cerr << "Unexpected end of input in argument list.\n"; return std::nullopt; @@ -415,7 +418,7 @@ class Parser { Expression &expr = arg.expr; if (auto *ident = std::get_if(&token)) { - auto next = next_non_newline_token(); + auto next = next_interesting_token(); if (!next) { std::cerr << "Unexpected end of input in argument list.\n"; return std::nullopt; @@ -425,7 +428,7 @@ class Parser { seen_kw_arg = true; name = Identifier{.name = std::move(ident->name)}; - auto value_token = next_non_newline_token(); + auto value_token = next_interesting_token(); if (!value_token) { std::cerr << "Unexpected end of input in argument list.\n"; return std::nullopt; diff --git a/starlark/parser_test.cc b/starlark/parser_test.cc index ab0bede..32d2441 100644 --- a/starlark/parser_test.cc +++ b/starlark/parser_test.cc @@ -257,7 +257,7 @@ int main() { }, }, { - "[\nx\n,\ny\n,\nz\n]", + "[\nx\n, \n y \n ,\nz\n]", starlark::Program{ .statements{ starlark::ExpressionStmt{ @@ -507,6 +507,9 @@ int main() { // ExpressionStmt // Statements must be newline-separated. "42 42", + + // Bad indentation. + " x = 1", }); etest::Suite s{}; diff --git a/starlark/token.h b/starlark/token.h index 951ddca..ac3ebf1 100644 --- a/starlark/token.h +++ b/starlark/token.h @@ -326,6 +326,18 @@ struct StringLiteral { inline std::string to_string(StringLiteral const &str) { return std::format(R"("{}")", str.value); } +struct Indent { + constexpr bool operator==(Indent const &) const = default; +}; + +constexpr std::string_view to_string(Indent const &) { return ""; } + +struct Dedent { + constexpr bool operator==(Dedent const &) const = default; +}; + +constexpr std::string_view to_string(Dedent const &) { return ""; } + struct Newline { constexpr bool operator==(Newline const &) const = default; }; @@ -400,6 +412,8 @@ using Token = std::variant< token::Identifier, token::IntLiteral, token::StringLiteral, + token::Indent, + token::Dedent, token::Newline, token::Eof>; diff --git a/starlark/tokenizer.h b/starlark/tokenizer.h index 079cb3d..4d0bd76 100644 --- a/starlark/tokenizer.h +++ b/starlark/tokenizer.h @@ -27,14 +27,54 @@ class Tokenizer { explicit Tokenizer(std::string_view input) : input_(input) {} std::optional tokenize() { + if (!pending_tokens_.empty()) { + auto token = pending_tokens_.back(); + pending_tokens_.pop_back(); + return token; + } + + // If we are starting a new line and find a non-whitespace/comment + // character, we check for an indentation change. + while (std::exchange(at_line_start_, false)) { + std::size_t current_indent = 0; + while (pos_ < input_.size() && input_[pos_] == ' ') { + ++current_indent; + ++pos_; + } + + if (pos_ < input_.size() && input_[pos_] == '#') { + while (pos_ < input_.size() && input_[pos_] != '\n') { + ++pos_; + } + + ++pos_; + at_line_start_ = true; + } else if (pos_ < input_.size() && input_[pos_] != '\n') { + handle_indent_change(current_indent); + + if (!pending_tokens_.empty()) { + auto token = pending_tokens_.back(); + pending_tokens_.pop_back(); + return token; + } + } + } + skip_comments_and_whitespace(); + // At EOF, emit dedents for all remaining indentation levels. if (pos_ >= input_.size()) { + if (indent_stack_.size() > 1) { + indent_stack_.pop_back(); + return token::Dedent{}; + } + return token::Eof{}; } if (input_[pos_] == '\n') { ++pos_; + at_line_start_ = true; return token::Newline{}; } @@ -91,6 +131,21 @@ class Tokenizer { } } + void handle_indent_change(std::size_t current_indent) { + assert(!indent_stack_.empty()); + std::size_t previous_indent = indent_stack_.back(); + + if (current_indent > previous_indent) { + indent_stack_.push_back(current_indent); + pending_tokens_.push_back(token::Indent{}); + } else if (current_indent < previous_indent) { + while (!indent_stack_.empty() && indent_stack_.back() > current_indent) { + indent_stack_.pop_back(); + pending_tokens_.push_back(token::Dedent{}); + } + } + } + std::optional tokenize_number() { assert(is_digit(input_[pos_]) || input_[pos_] == '-'); @@ -289,6 +344,9 @@ class Tokenizer { std::string_view input_; std::size_t pos_ = 0; + std::vector indent_stack_{0}; + std::vector pending_tokens_; + bool at_line_start_{true}; }; inline std::optional> tokenize(std::string_view input) { diff --git a/starlark/tokenizer_test.cc b/starlark/tokenizer_test.cc index 9794c00..06be1d6 100644 --- a/starlark/tokenizer_test.cc +++ b/starlark/tokenizer_test.cc @@ -76,6 +76,115 @@ world''')", t::Identifier{"world"}, }, }, + { + "foo\n bar", + Tokens{ + t::Identifier{"foo"}, + t::Newline{}, + t::Indent{}, + t::Identifier{"bar"}, + t::Dedent{}, + }, + }, + { + "foo\n bar", + Tokens{ + t::Identifier{"foo"}, + t::Newline{}, + t::Indent{}, + t::Identifier{"bar"}, + t::Dedent{}, + }, + }, + { + "foo\n bar\nbaz", + Tokens{ + t::Identifier{"foo"}, + t::Newline{}, + t::Indent{}, + t::Identifier{"bar"}, + t::Newline{}, + t::Dedent{}, + t::Identifier{"baz"}, + }, + }, + { + "foo\n bar\n baz\nqux", + Tokens{ + t::Identifier{"foo"}, + t::Newline{}, + t::Indent{}, + t::Identifier{"bar"}, + t::Newline{}, + t::Identifier{"baz"}, + t::Newline{}, + t::Dedent{}, + t::Identifier{"qux"}, + }, + }, + { + "foo\n bar\n baz\n qux", + Tokens{ + t::Identifier{"foo"}, + t::Newline{}, + t::Indent{}, + t::Identifier{"bar"}, + t::Newline{}, + t::Indent{}, + t::Identifier{"baz"}, + t::Newline{}, + t::Dedent{}, + t::Identifier{"qux"}, + t::Dedent{}, + }, + }, + { + "foo\n bar\n baz\nqux", + Tokens{ + t::Identifier{"foo"}, + t::Newline{}, + t::Indent{}, + t::Identifier{"bar"}, + t::Newline{}, + t::Indent{}, + t::Identifier{"baz"}, + t::Newline{}, + t::Dedent{}, + t::Dedent{}, + t::Identifier{"qux"}, + }, + }, + { + "foo\n bar\n\nbaz", + Tokens{ + t::Identifier{"foo"}, + t::Newline{}, + t::Indent{}, + t::Identifier{"bar"}, + t::Newline{}, + t::Newline{}, + t::Dedent{}, + t::Identifier{"baz"}, + }, + }, + { + "foo\nbar # hello\n# comment mccommentface\nbaz", + Tokens{ + t::Identifier{"foo"}, + t::Newline{}, + t::Identifier{"bar"}, + t::Newline{}, + t::Identifier{"baz"}, + }, + }, + { + "foo\n # hello\nbar", + Tokens{ + t::Identifier{"foo"}, + t::Newline{}, + t::Identifier{"bar"}, + }, + }, }); etest::Suite s{};