Skip to content

Commit c7b22a9

Browse files
KobzolKobzol
committed
Pass TokenArena to Parser 
1 parent 0084ccc commit c7b22a9

14 files changed

Lines changed: 60 additions & 28 deletions

File tree

‎compiler/rustc_ast/src/tokenarena.rs‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -60,6 +60,10 @@ impl TokenArena {
6060
self.tokens.len()
6161
}
6262

63+
pub fn is_empty(&self) -> bool {
64+
self.tokens.is_empty()
65+
}
66+
6367
pub fn from_stream(stream: &TokenStream) -> Self {
6468
let mut arena = TokenArena { tokens: Vec::with_capacity(stream.len()) };
6569
arena.fill(stream);

‎compiler/rustc_ast/src/tokenstream.rs‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@ use thin_vec::ThinVec;
2020
use crate::ast::AttrStyle;
2121
use crate::ast_traits::HasTokens;
2222
use crate::token::{self, Delimiter, Token, TokenKind};
23+
use crate::tokenarena::TokenArena;
2324
use crate::{AttrVec, Attribute};
2425

2526
#[cfg(test)]
@@ -949,7 +950,8 @@ pub struct TokenCursor {
949950

950951
impl TokenCursor {
951952
#[inline]
952-
pub fn new(stream: TokenStream) -> Self {
953+
pub fn new(arena: TokenArena) -> Self {
954+
let stream = arena.to_token_stream();
953955
TokenCursor { curr: TokenTreeCursor::new(stream), stack: vec![] }
954956
}
955957

‎compiler/rustc_attr_parsing/src/attributes/cfg.rs‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
use std::convert::identity;
22

33
use rustc_ast::token::Delimiter;
4+
use rustc_ast::tokenarena::TokenArena;
45
use rustc_ast::tokenstream::{DelimSpan, WithTokens};
56
use rustc_ast::{AttrItem, Attribute, LitKind, ast, token};
67
use rustc_attr_ir::target::Target;
@@ -312,7 +313,7 @@ pub fn parse_cfg_attr(
312313
match &cfg_attr.get_normal_item().args {
313314
ast::AttrArgs::Delimited(ast::DelimArgs { dspan, delim, tokens }) if !tokens.is_empty() => {
314315
check_cfg_attr_bad_delim(&sess.psess, *dspan, *delim);
315-
match parse_in(&sess.psess, tokens.clone(), "`cfg_attr` input", |p| {
316+
match parse_in(&sess.psess, TokenArena::from_stream(tokens), "`cfg_attr` input", |p| {
316317
parse_cfg_attr_internal(p, sess, features, lint_node_id, cfg_attr)
317318
}) {
318319
Ok(r) => return Some(r),

‎compiler/rustc_attr_parsing/src/parser.rs‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@ use std::fmt::{Debug, Display};
1414
use std::sync::atomic::{AtomicBool, Ordering};
1515

1616
use rustc_ast::token::{self, Delimiter, MetaVarKind};
17+
use rustc_ast::tokenarena::TokenArena;
1718
use rustc_ast::tokenstream::TokenStream;
1819
use rustc_ast::{
1920
AttrArgs, Expr, ExprKind, LitKind, MetaItemLit, Path, PathSegment, StmtKind, UnOp,
@@ -721,13 +722,13 @@ impl<'a, 'sess> MetaItemListParserContext<'a, 'sess> {
721722
}
722723

723724
fn parse(
724-
tokens: TokenStream,
725+
arena: TokenArena,
725726
psess: &'sess ParseSess,
726727
span: Span,
727728
should_emit: ShouldEmit,
728729
allow_expr_metavar: AllowExprMetavar,
729730
) -> PResult<'sess, MetaItemListParser> {
730-
let mut parser = Parser::new(psess, tokens, None);
731+
let mut parser = Parser::new(psess, arena, None);
731732
if let ShouldEmit::ErrorsAndLints { recovery } = should_emit {
732733
parser = parser.recovery(recovery);
733734
}
@@ -764,7 +765,7 @@ impl MetaItemListParser {
764765
allow_expr_metavar: AllowExprMetavar,
765766
) -> Result<Self, Diag<'sess>> {
766767
MetaItemListParserContext::parse(
767-
tokens.clone(),
768+
TokenArena::from_stream(tokens),
768769
psess,
769770
span,
770771
should_emit,

‎compiler/rustc_attr_parsing/src/validate_attr.rs‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@ use std::convert::identity;
44
use std::slice;
55

66
use rustc_ast::token::Delimiter;
7+
use rustc_ast::tokenarena::TokenArena;
78
use rustc_ast::tokenstream::DelimSpan;
89
use rustc_ast::{
910
self as ast, AttrArgs, AttrKind, Attribute, DelimArgs, MetaItem, MetaItemInner, MetaItemKind,
@@ -75,8 +76,9 @@ pub fn parse_meta<'a>(psess: &'a ParseSess, attr: &Attribute) -> PResult<'a, Met
7576
AttrArgs::Empty => MetaItemKind::Word,
7677
AttrArgs::Delimited(DelimArgs { dspan, delim, tokens }) => {
7778
check_meta_bad_delim(psess, *dspan, *delim);
78-
let nmis =
79-
parse_in(psess, tokens.clone(), "meta list", |p| p.parse_meta_seq_top())?;
79+
let nmis = parse_in(psess, TokenArena::from_stream(tokens), "meta list", |p| {
80+
p.parse_meta_seq_top()
81+
})?;
8082
MetaItemKind::List(nmis)
8183
}
8284
AttrArgs::Eq { expr, .. } => {

‎compiler/rustc_builtin_macros/src/cfg_eval.rs‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@ use core::ops::ControlFlow;
22

33
use rustc_ast as ast;
44
use rustc_ast::mut_visit::MutVisitor;
5+
use rustc_ast::tokenarena::TokenArena;
56
use rustc_ast::visit::{AssocCtxt, Visitor};
67
use rustc_ast::{Attribute, HasTokens, NodeId, mut_visit, visit};
78
use rustc_errors::PResult;
@@ -105,7 +106,8 @@ impl CfgEval<'_> {
105106
//
106107
// After that we have our re-parsed `AttrTokenStream`, recursively configuring
107108
// our attribute target will correctly configure the tokens as well.
108-
let mut parser = Parser::new(&self.0.sess.psess, orig_tokens, None);
109+
let mut parser =
110+
Parser::new(&self.0.sess.psess, TokenArena::from_stream(&orig_tokens), None);
109111
parser.capture_cfg = true;
110112
let res: PResult<'_, Option<Annotatable>> = try {
111113
match &annotatable {

‎compiler/rustc_expand/src/base.rs‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@ use std::rc::Rc;
66
use std::sync::Arc;
77

88
use rustc_ast::attr::MarkedAttrs;
9+
use rustc_ast::tokenarena::TokenArena;
910
use rustc_ast::tokenstream::TokenStream;
1011
use rustc_ast::visit::{AssocCtxt, Visitor};
1112
use rustc_ast::{self as ast, AttrVec, Attribute, HasAttrs, Item, NodeId, PatKind, Safety};
@@ -1258,7 +1259,7 @@ impl<'a> ExtCtxt<'a> {
12581259
expand::MacroExpander::new(self, true)
12591260
}
12601261
pub fn new_parser_from_tts(&self, stream: TokenStream) -> Parser<'a> {
1261-
Parser::new(&self.sess.psess, stream, MACRO_ARGUMENTS)
1262+
Parser::new(&self.sess.psess, TokenArena::from_stream(&stream), MACRO_ARGUMENTS)
12621263
}
12631264
pub fn source_map(&self) -> &'a SourceMap {
12641265
self.sess.psess.source_map()

‎compiler/rustc_expand/src/expand.rs‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -777,6 +777,7 @@ impl<'a, 'b> MacroExpander<'a, 'b> {
777777
// we are invoking it on an out-of-line module or crate.
778778
Annotatable::Crate(krate) => {
779779
rustc_parse::fake_token_stream_for_crate(&self.cx.sess.psess, krate)
780+
.to_token_stream()
780781
}
781782
Annotatable::Item(item_inner)
782783
if matches!(attr.style, AttrStyle::Inner)
@@ -795,13 +796,15 @@ impl<'a, 'b> MacroExpander<'a, 'b> {
795796
item_inner,
796797
Some(&attr),
797798
)
799+
.to_token_stream()
798800
}
799801
Annotatable::Item(item_inner) if item_inner.tokens.is_none() => {
800802
rustc_parse::fake_token_stream_for_item(
801803
&self.cx.sess.psess,
802804
item_inner,
803805
None,
804806
)
807+
.to_token_stream()
805808
}
806809
// When a function has EII implementations attached (via `eii_impl`),
807810
// use fake tokens so the pretty-printer re-emits the EII attribute
@@ -818,12 +821,14 @@ impl<'a, 'b> MacroExpander<'a, 'b> {
818821
item_inner,
819822
None,
820823
)
824+
.to_token_stream()
821825
}
822826
Annotatable::ForeignItem(item_inner) if item_inner.tokens.is_none() => {
823827
rustc_parse::fake_token_stream_for_foreign_item(
824828
&self.cx.sess.psess,
825829
item_inner,
826830
)
831+
.to_token_stream()
827832
}
828833
_ => item.to_tokens(),
829834
};

‎compiler/rustc_expand/src/mbe/macro_rules.rs‎

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@ use ast::token::IdentIsRaw;
77
use rustc_ast::token::NtPatKind::*;
88
use rustc_ast::token::TokenKind::*;
99
use rustc_ast::token::{self, Delimiter, NonterminalKind, Token, TokenKind};
10+
use rustc_ast::tokenarena::TokenArena;
1011
use rustc_ast::tokenstream::{self, DelimSpan, TokenStream};
1112
use rustc_ast::{self as ast, DUMMY_NODE_ID, NodeId, Safety};
1213
use rustc_ast_pretty::pprust;
@@ -132,7 +133,7 @@ impl<'a, 'b> ParserAnyMacro<'a, 'b> {
132133
matched_rule_bindings: &'b [MatcherLoc],
133134
) -> Self {
134135
Self {
135-
parser: Parser::new(&cx.sess.psess, tts, None),
136+
parser: Parser::new(&cx.sess.psess, TokenArena::from_stream(&tts), None),
136137

137138
// Pass along the original expansion site and the name of the macro
138139
// so we can print a useful error message if the parse of the expanded
@@ -796,7 +797,7 @@ pub fn compile_declarative_macro(
796797
let macro_rules = macro_def.macro_rules;
797798
let exp_sep = if macro_rules { exp!(Semi) } else { exp!(Comma) };
798799

799-
let body = macro_def.body.tokens.clone();
800+
let body = TokenArena::from_stream(&macro_def.body.tokens);
800801
let mut p = Parser::new(&sess.psess, body, rustc_parse::MACRO_ARGUMENTS);
801802

802803
// Don't abort iteration early, so that multiple errors can be reported. We only abort early on
@@ -1869,5 +1870,6 @@ pub(super) fn parser_from_cx(
18691870
recovery: Recovery,
18701871
) -> Parser<'_> {
18711872
tts.desugar_doc_comments();
1872-
Parser::new(psess, tts, rustc_parse::MACRO_ARGUMENTS).recovery(recovery)
1873+
Parser::new(psess, TokenArena::from_stream(&tts), rustc_parse::MACRO_ARGUMENTS)
1874+
.recovery(recovery)
18731875
}

‎compiler/rustc_expand/src/proc_macro.rs‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
use rustc_ast as ast;
2+
use rustc_ast::tokenarena::TokenArena;
23
use rustc_ast::tokenstream::TokenStream;
34
use rustc_data_structures::profiling::TimingGuard;
45
use rustc_errors::ErrorGuaranteed;
@@ -131,7 +132,11 @@ impl MultiItemModifier for DeriveProcMacro {
131132
};
132133

133134
let error_count_before = ecx.dcx().err_count();
134-
let mut parser = Parser::new(&ecx.sess.psess, output, Some("proc-macro derive"));
135+
let mut parser = Parser::new(
136+
&ecx.sess.psess,
137+
TokenArena::from_stream(&output),
138+
Some("proc-macro derive"),
139+
);
135140
let mut items = vec![];
136141

137142
loop {

0 commit comments

Comments
 (0)