Skip to content

starlark: Treat newlines as tokens #46

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
robinlinden merged 1 commit into from
May 18, 2026
Merged

starlark: Treat newlines as tokens #46

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
58 changes: 44 additions & 14 deletions starlark/parser.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@ class Parser {

std::optional<Program> parse() {
Program program;
bool needs_newline = false;

for (auto maybe_token = next_token(); maybe_token; maybe_token = next_token()) {
auto &token = *maybe_token;
Expand All @@ -33,6 +34,22 @@ class Parser {
return program;
}

if (std::exchange(needs_newline, false)) {
if (!std::holds_alternative<token::Newline>(token)) {
std::cerr << "Expected newline after previous statement, but got: "
<< to_string(token) << '\n';
return std::nullopt;
}

continue;
}

if (std::holds_alternative<token::Newline>(token)) {
continue;
}

needs_newline = true;

if (std::holds_alternative<token::Load>(token)) {
auto load = parse_load_stmt();
if (!load) {
Expand Down Expand Up @@ -97,6 +114,19 @@ class Parser {
return tokenizer_.tokenize();
}

std::optional<Token> next_non_newline_token() {
while (true) {
auto token = next_token();
if (!token) {
return std::nullopt;
}

if (!std::holds_alternative<token::Newline>(*token)) {
return token;
}
}
}

void reconsume(Token token) { peeked_token_ = std::move(token); }

std::optional<Expression> parse_operand(Token &token) {
Expand All @@ -115,7 +145,7 @@ class Parser {
if (std::holds_alternative<token::LBracket>(token)) {
std::vector<Expression> elements;
while (true) {
auto maybe_token = next_token();
auto maybe_token = next_non_newline_token();
if (!maybe_token) {
std::cerr << "Tokenization error in list expression.\n";
return std::nullopt;
Expand All @@ -134,22 +164,22 @@ class Parser {
}

// On the first iteration, we check if this is a list comprehension.
auto maybe_next = next_token();
auto maybe_next = next_non_newline_token();
Comment thread
robinlinden marked this conversation as resolved.
if (elements.empty() && maybe_next.has_value() &&
std::holds_alternative<token::For>(*maybe_next)) {
auto var_token = next_token();
auto var_token = next_non_newline_token();
if (!var_token || !std::holds_alternative<token::Identifier>(*var_token)) {
std::cerr << "Expected identifier in list comprehension.\n";
return std::nullopt;
}

auto in_token = next_token();
auto in_token = next_non_newline_token();
if (!in_token || !std::holds_alternative<token::In>(*in_token)) {
std::cerr << "Expected 'in' in list comprehension.\n";
return std::nullopt;
}

auto iterable_token = next_token();
auto iterable_token = next_non_newline_token();
if (!iterable_token) {
std::cerr << "Unexpected end of input in list comprehension.\n";
return std::nullopt;
Expand All @@ -161,7 +191,7 @@ class Parser {
return std::nullopt;
}

auto maybe_closing = next_token();
auto maybe_closing = next_non_newline_token();
if (!maybe_closing ||
!std::holds_alternative<token::RBracket>(*maybe_closing)) {
std::cerr << "Expected closing ']' in list comprehension.\n";
Expand Down Expand Up @@ -206,7 +236,7 @@ class Parser {
std::vector<std::pair<Expression, Expression>> entries;

while (true) {
auto maybe_token = next_token();
auto maybe_token = next_non_newline_token();
if (!maybe_token) {
std::cerr << "Tokenization error in dict expression.\n";
return std::nullopt;
Expand All @@ -224,13 +254,13 @@ class Parser {
return std::nullopt;
}

auto colon_token = next_token();
auto colon_token = next_non_newline_token();
if (!colon_token || !std::holds_alternative<token::Colon>(*colon_token)) {
std::cerr << "Expected ':' after dict key.\n";
return std::nullopt;
}

auto value_token = next_token();
auto value_token = next_non_newline_token();
if (!value_token) {
std::cerr << "Unexpected end of input in dict expression.\n";
return std::nullopt;
Expand All @@ -244,7 +274,7 @@ class Parser {

entries.emplace_back(std::move(*key_expr), std::move(*value_expr));

auto maybe_next = next_token();
auto maybe_next = next_non_newline_token();
if (!maybe_next) {
std::cerr << "Tokenization error in dict expression.\n";
return std::nullopt;
Expand Down Expand Up @@ -350,7 +380,7 @@ class Parser {
bool seen_kw_arg = false;

while (true) {
auto maybe_token = next_token();
auto maybe_token = next_non_newline_token();
if (!maybe_token) {
std::cerr << "Unexpected end of input in argument list.\n";
return std::nullopt;
Expand All @@ -366,7 +396,7 @@ class Parser {
}

if (std::holds_alternative<token::Comma>(token)) {
maybe_token = next_token();
maybe_token = next_non_newline_token();
if (!maybe_token) {
std::cerr << "Unexpected end of input in argument list.\n";
return std::nullopt;
Expand All @@ -385,7 +415,7 @@ class Parser {
Expression &expr = arg.expr;

if (auto *ident = std::get_if<token::Identifier>(&token)) {
auto next = next_token();
auto next = next_non_newline_token();
if (!next) {
std::cerr << "Unexpected end of input in argument list.\n";
return std::nullopt;
Expand All @@ -395,7 +425,7 @@ class Parser {
seen_kw_arg = true;
name = Identifier{.name = std::move(ident->name)};

auto value_token = next_token();
auto value_token = next_non_newline_token();
if (!value_token) {
std::cerr << "Unexpected end of input in argument list.\n";
return std::nullopt;
Expand Down
124 changes: 124 additions & 0 deletions starlark/parser_test.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -167,6 +167,33 @@ int main() {
},
},
},
{
"{\n42\n:\nbar\n,\nbaz()\n:\n13\n}",
starlark::Program{
.statements{
starlark::ExpressionStmt{
.expr{
starlark::DictExpr{
.entries{
{
starlark::IntLiteral{42},
starlark::Identifier{"bar"},
},
{
starlark::CallExpr{
.target = std::make_shared<starlark::Expression>(
starlark::Identifier{"baz"}),
.args{},
},
starlark::IntLiteral{13},
},
},
},
},
},
},
},
},
{
"{}",
starlark::Program{
Expand All @@ -181,6 +208,72 @@ int main() {
},
},
},
{
"[]",
starlark::Program{
.statements{
starlark::ExpressionStmt{
.expr{
starlark::ListExpr{
.elements{},
},
},
},
},
},
},
{
R"(["foo"])",
starlark::Program{
.statements{
starlark::ExpressionStmt{
.expr{
starlark::ListExpr{
.elements{
starlark::Expression{starlark::StringLiteral{"foo"}},
},
},
},
},
},
},
},
{
"[x, y, z]",
starlark::Program{
.statements{
starlark::ExpressionStmt{
.expr{
starlark::ListExpr{
.elements{
starlark::Expression{starlark::Identifier{"x"}},
starlark::Expression{starlark::Identifier{"y"}},
starlark::Expression{starlark::Identifier{"z"}},
},
},
},
},
},
},
},
{
"[\nx\n,\ny\n,\nz\n]",
starlark::Program{
.statements{
starlark::ExpressionStmt{
.expr{
starlark::ListExpr{
.elements{
starlark::Expression{starlark::Identifier{"x"}},
starlark::Expression{starlark::Identifier{"y"}},
starlark::Expression{starlark::Identifier{"z"}},
},
},
},
},
},
},
},
{
"[x for x in y]",
starlark::Program{
Expand Down Expand Up @@ -259,6 +352,16 @@ int main() {
},
},
},
{
"42\n",
starlark::Program{
.statements{
starlark::ExpressionStmt{
.expr{starlark::IntLiteral{42}},
},
},
},
},
{
"42[5]",
starlark::Program{
Expand All @@ -276,6 +379,23 @@ int main() {
},
},
},
{
"42\n[5]",
starlark::Program{
.statements{
starlark::ExpressionStmt{
.expr{starlark::IntLiteral{42}},
},
starlark::ExpressionStmt{.expr{
starlark::ListExpr{
.elements{
starlark::Expression{starlark::IntLiteral{5}},
},
},
}},
},
},
},
{
"foo.bar",
starlark::Program{
Expand Down Expand Up @@ -383,6 +503,10 @@ int main() {
"foo.",
// Invalid member name.
"foo.5",

// ExpressionStmt
// Statements must be newline-separated.
"42 42",
});

etest::Suite s{};
Expand Down
7 changes: 7 additions & 0 deletions starlark/token.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -326,6 +326,12 @@ struct StringLiteral {

inline std::string to_string(StringLiteral const &str) { return std::format(R"("{}")", str.value); }

struct Newline {
constexpr bool operator==(Newline const &) const = default;
};

constexpr std::string_view to_string(Newline const &) { return "<newline>"; }

struct Eof {
constexpr bool operator==(Eof const &) const = default;
};
Expand Down Expand Up @@ -394,6 +400,7 @@ using Token = std::variant<
token::Identifier,
token::IntLiteral,
token::StringLiteral,
token::Newline,
token::Eof>;

inline std::string to_string(Token const &token) {
Expand Down
7 changes: 6 additions & 1 deletion starlark/tokenizer.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,11 @@ class Tokenizer {
return token::Eof{};
}

if (input_[pos_] == '\n') {
++pos_;
return token::Newline{};
}

if (input_.substr(pos_, 3) == R"(""")") {
return tokenize_multiline_string('"');
}
Expand Down Expand Up @@ -62,7 +67,7 @@ class Tokenizer {
}

private:
bool is_whitespace(char c) const { return c == ' ' || c == '\t' || c == '\n' || c == '\r'; }
bool is_whitespace(char c) const { return c == ' ' || c == '\t' || c == '\r'; }

bool is_alpha(char c) const {
return (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || c == '_';
Expand Down
8 changes: 8 additions & 0 deletions starlark/tokenizer_test.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -68,6 +68,14 @@ world""")",
world''')",
Tokens{t::StringLiteral{"hello\nworld"}},
},
{
"hello\nworld",
Tokens{
t::Identifier{"hello"},
t::Newline{},
t::Identifier{"world"},
},
},
});

etest::Suite s{};
Expand Down
Loading