Skip to content

starlark: Implement indentation tokenization #51

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
robinlinden merged 1 commit into from
Aug 21, 2026
Merged

starlark: Implement indentation tokenization #51

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
35 changes: 19 additions & 16 deletions starlark/parser.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -114,14 +114,17 @@ class Parser {
return tokenizer_.tokenize();
}

std::optional<Token> next_non_newline_token() {
// Starlark ignores things like newlines and indents in some contexts.
std::optional<Token> next_interesting_token() {
while (true) {
auto token = next_token();
if (!token) {
return std::nullopt;
}

if (!std::holds_alternative<token::Newline>(*token)) {
if (!std::holds_alternative<token::Newline>(*token) &&
!std::holds_alternative<token::Indent>(*token) &&
!std::holds_alternative<token::Dedent>(*token)) {
return token;
}
}
Expand All @@ -145,7 +148,7 @@ class Parser {
if (std::holds_alternative<token::LBracket>(token)) {
std::vector<Expression> elements;
while (true) {
auto maybe_token = next_non_newline_token();
auto maybe_token = next_interesting_token();
if (!maybe_token) {
std::cerr << "Tokenization error in list expression.\n";
return std::nullopt;
Expand All @@ -164,22 +167,22 @@ class Parser {
}

// On the first iteration, we check if this is a list comprehension.
auto maybe_next = next_non_newline_token();
auto maybe_next = next_interesting_token();
if (elements.empty() && maybe_next.has_value() &&
std::holds_alternative<token::For>(*maybe_next)) {
auto var_token = next_non_newline_token();
auto var_token = next_interesting_token();
if (!var_token || !std::holds_alternative<token::Identifier>(*var_token)) {
std::cerr << "Expected identifier in list comprehension.\n";
return std::nullopt;
}

auto in_token = next_non_newline_token();
auto in_token = next_interesting_token();
if (!in_token || !std::holds_alternative<token::In>(*in_token)) {
std::cerr << "Expected 'in' in list comprehension.\n";
return std::nullopt;
}

auto iterable_token = next_non_newline_token();
auto iterable_token = next_interesting_token();
if (!iterable_token) {
std::cerr << "Unexpected end of input in list comprehension.\n";
return std::nullopt;
Expand All @@ -191,7 +194,7 @@ class Parser {
return std::nullopt;
}

auto maybe_closing = next_non_newline_token();
auto maybe_closing = next_interesting_token();
if (!maybe_closing ||
!std::holds_alternative<token::RBracket>(*maybe_closing)) {
std::cerr << "Expected closing ']' in list comprehension.\n";
Expand Down Expand Up @@ -236,7 +239,7 @@ class Parser {
std::vector<std::pair<Expression, Expression>> entries;

while (true) {
auto maybe_token = next_non_newline_token();
auto maybe_token = next_interesting_token();
if (!maybe_token) {
std::cerr << "Tokenization error in dict expression.\n";
return std::nullopt;
Expand All @@ -254,13 +257,13 @@ class Parser {
return std::nullopt;
}

auto colon_token = next_non_newline_token();
auto colon_token = next_interesting_token();
if (!colon_token || !std::holds_alternative<token::Colon>(*colon_token)) {
std::cerr << "Expected ':' after dict key.\n";
return std::nullopt;
}

auto value_token = next_non_newline_token();
auto value_token = next_interesting_token();
if (!value_token) {
std::cerr << "Unexpected end of input in dict expression.\n";
return std::nullopt;
Expand All @@ -274,7 +277,7 @@ class Parser {

entries.emplace_back(std::move(*key_expr), std::move(*value_expr));

auto maybe_next = next_non_newline_token();
auto maybe_next = next_interesting_token();
if (!maybe_next) {
std::cerr << "Tokenization error in dict expression.\n";
return std::nullopt;
Expand Down Expand Up @@ -380,7 +383,7 @@ class Parser {
bool seen_kw_arg = false;

while (true) {
auto maybe_token = next_non_newline_token();
auto maybe_token = next_interesting_token();
if (!maybe_token) {
std::cerr << "Unexpected end of input in argument list.\n";
return std::nullopt;
Expand All @@ -396,7 +399,7 @@ class Parser {
}

if (std::holds_alternative<token::Comma>(token)) {
maybe_token = next_non_newline_token();
maybe_token = next_interesting_token();
if (!maybe_token) {
std::cerr << "Unexpected end of input in argument list.\n";
return std::nullopt;
Expand All @@ -415,7 +418,7 @@ class Parser {
Expression &expr = arg.expr;

if (auto *ident = std::get_if<token::Identifier>(&token)) {
auto next = next_non_newline_token();
auto next = next_interesting_token();
if (!next) {
std::cerr << "Unexpected end of input in argument list.\n";
return std::nullopt;
Expand All @@ -425,7 +428,7 @@ class Parser {
seen_kw_arg = true;
name = Identifier{.name = std::move(ident->name)};

auto value_token = next_non_newline_token();
auto value_token = next_interesting_token();
if (!value_token) {
std::cerr << "Unexpected end of input in argument list.\n";
return std::nullopt;
Expand Down
5 changes: 4 additions & 1 deletion starlark/parser_test.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -257,7 +257,7 @@ int main() {
},
},
{
"[\nx\n,\ny\n,\nz\n]",
"[\nx\n, \n y \n ,\nz\n]",
starlark::Program{
.statements{
starlark::ExpressionStmt{
Expand Down Expand Up @@ -507,6 +507,9 @@ int main() {
// ExpressionStmt
// Statements must be newline-separated.
"42 42",

// Bad indentation.
" x = 1",
});

etest::Suite s{};
Expand Down
14 changes: 14 additions & 0 deletions starlark/token.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -326,6 +326,18 @@ struct StringLiteral {

inline std::string to_string(StringLiteral const &str) { return std::format(R"("{}")", str.value); }

struct Indent {
constexpr bool operator==(Indent const &) const = default;
};

constexpr std::string_view to_string(Indent const &) { return "<indent>"; }

struct Dedent {
constexpr bool operator==(Dedent const &) const = default;
};

constexpr std::string_view to_string(Dedent const &) { return "<dedent>"; }

struct Newline {
constexpr bool operator==(Newline const &) const = default;
};
Expand Down Expand Up @@ -400,6 +412,8 @@ using Token = std::variant<
token::Identifier,
token::IntLiteral,
token::StringLiteral,
token::Indent,
token::Dedent,
token::Newline,
token::Eof>;

Expand Down
58 changes: 58 additions & 0 deletions starlark/tokenizer.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,14 +27,54 @@ class Tokenizer {
explicit Tokenizer(std::string_view input) : input_(input) {}

std::optional<Token> tokenize() {
if (!pending_tokens_.empty()) {
auto token = pending_tokens_.back();
pending_tokens_.pop_back();
return token;
}

// If we are starting a new line and find a non-whitespace/comment
// character, we check for an indentation change.
while (std::exchange(at_line_start_, false)) {
std::size_t current_indent = 0;
while (pos_ < input_.size() && input_[pos_] == ' ') {
++current_indent;
++pos_;
}

if (pos_ < input_.size() && input_[pos_] == '#') {
while (pos_ < input_.size() && input_[pos_] != '\n') {
++pos_;
}

++pos_;
at_line_start_ = true;
} else if (pos_ < input_.size() && input_[pos_] != '\n') {
handle_indent_change(current_indent);

if (!pending_tokens_.empty()) {
auto token = pending_tokens_.back();
pending_tokens_.pop_back();
return token;
}
}
}

skip_comments_and_whitespace();

// At EOF, emit dedents for all remaining indentation levels.
if (pos_ >= input_.size()) {
if (indent_stack_.size() > 1) {
indent_stack_.pop_back();
return token::Dedent{};
}

return token::Eof{};
}

if (input_[pos_] == '\n') {
++pos_;
at_line_start_ = true;
return token::Newline{};
}

Expand Down Expand Up @@ -91,6 +131,21 @@ class Tokenizer {
}
}

void handle_indent_change(std::size_t current_indent) {
assert(!indent_stack_.empty());
std::size_t previous_indent = indent_stack_.back();

if (current_indent > previous_indent) {
indent_stack_.push_back(current_indent);
pending_tokens_.push_back(token::Indent{});
} else if (current_indent < previous_indent) {
while (!indent_stack_.empty() && indent_stack_.back() > current_indent) {
indent_stack_.pop_back();
pending_tokens_.push_back(token::Dedent{});
}
}
}

std::optional<Token> tokenize_number() {
assert(is_digit(input_[pos_]) || input_[pos_] == '-');

Expand Down Expand Up @@ -289,6 +344,9 @@ class Tokenizer {

std::string_view input_;
std::size_t pos_ = 0;
std::vector<std::size_t> indent_stack_{0};
std::vector<Token> pending_tokens_;
bool at_line_start_{true};
};

inline std::optional<std::vector<Token>> tokenize(std::string_view input) {
Expand Down
109 changes: 109 additions & 0 deletions starlark/tokenizer_test.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -76,6 +76,115 @@ world''')",
t::Identifier{"world"},
},
},
{
"foo\n bar",
Tokens{
t::Identifier{"foo"},
t::Newline{},
t::Indent{},
t::Identifier{"bar"},
t::Dedent{},
},
},
{
"foo\n bar",
Tokens{
t::Identifier{"foo"},
t::Newline{},
t::Indent{},
t::Identifier{"bar"},
t::Dedent{},
},
},
{
"foo\n bar\nbaz",
Tokens{
t::Identifier{"foo"},
t::Newline{},
t::Indent{},
t::Identifier{"bar"},
t::Newline{},
t::Dedent{},
t::Identifier{"baz"},
},
},
{
"foo\n bar\n baz\nqux",
Tokens{
t::Identifier{"foo"},
t::Newline{},
t::Indent{},
t::Identifier{"bar"},
t::Newline{},
t::Identifier{"baz"},
t::Newline{},
t::Dedent{},
t::Identifier{"qux"},
},
},
{
"foo\n bar\n baz\n qux",
Tokens{
t::Identifier{"foo"},
t::Newline{},
t::Indent{},
t::Identifier{"bar"},
t::Newline{},
t::Indent{},
t::Identifier{"baz"},
t::Newline{},
t::Dedent{},
t::Identifier{"qux"},
t::Dedent{},
},
},
{
"foo\n bar\n baz\nqux",
Tokens{
t::Identifier{"foo"},
t::Newline{},
t::Indent{},
t::Identifier{"bar"},
t::Newline{},
t::Indent{},
t::Identifier{"baz"},
t::Newline{},
t::Dedent{},
t::Dedent{},
t::Identifier{"qux"},
},
},
{
"foo\n bar\n\nbaz",
Tokens{
t::Identifier{"foo"},
t::Newline{},
t::Indent{},
t::Identifier{"bar"},
t::Newline{},
t::Newline{},
t::Dedent{},
t::Identifier{"baz"},
},
},
{
"foo\nbar # hello\n# comment mccommentface\nbaz",
Tokens{
t::Identifier{"foo"},
t::Newline{},
t::Identifier{"bar"},
t::Newline{},
t::Identifier{"baz"},
},
},
{
"foo\n # hello\nbar",
Tokens{
t::Identifier{"foo"},
t::Newline{},
t::Identifier{"bar"},
},
},
});

etest::Suite s{};
Expand Down