Skip to content

Commit 7c10b0d

Browse files
petrochenkovpetrochenkov
committed
[PERF-EXPERIMENT] Keep two spans per token
1 parent 95f6870 commit 7c10b0d

File tree

9 files changed

+32
-31
lines changed

9 files changed

+32
-31
lines changed

‎compiler/rustc_ast/src/attr/mod.rs

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -407,6 +407,7 @@ impl MetaItem {
407407
Some(TokenTree::Token(Token {
408408
kind: kind @ (token::Ident(..) | token::ModSep),
409409
span,
410+
..
410411
})) => 'arm: {
411412
let mut segments = if let token::Ident(name, _) = kind {
412413
if let Some(TokenTree::Token(Token { kind: token::ModSep, .. })) = tokens.peek()
@@ -420,8 +421,9 @@ impl MetaItem {
420421
vec![PathSegment::path_root(span)]
421422
};
422423
loop {
423-
if let Some(TokenTree::Token(Token { kind: token::Ident(name, _), span })) =
424-
tokens.next().map(TokenTree::uninterpolate)
424+
if let Some(TokenTree::Token(Token {
425+
kind: token::Ident(name, _), span, ..
426+
})) = tokens.next().map(TokenTree::uninterpolate)
425427
{
426428
segments.push(PathSegment::from_ident(Ident::new(name, span)));
427429
} else {

‎compiler/rustc_ast/src/mut_visit.rs

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -719,13 +719,14 @@ pub fn visit_lazy_tts<T: MutVisitor>(lazy_tts: &mut Option<LazyTokenStream>, vis
719719
// In practice the ident part is not actually used by specific visitors right now,
720720
// but there's a test below checking that it works.
721721
pub fn visit_token<T: MutVisitor>(t: &mut Token, vis: &mut T) {
722-
let Token { kind, span } = t;
722+
let Token { kind, span, span2 } = t;
723723
match kind {
724724
token::Ident(name, _) | token::Lifetime(name) => {
725725
let mut ident = Ident::new(*name, *span);
726726
vis.visit_ident(&mut ident);
727727
*name = ident.name;
728728
*span = ident.span;
729+
vis.visit_span(span2);
729730
return; // Avoid visiting the span for the second time.
730731
}
731732
token::Interpolated(nt) => {
@@ -735,6 +736,7 @@ pub fn visit_token<T: MutVisitor>(t: &mut Token, vis: &mut T) {
735736
_ => {}
736737
}
737738
vis.visit_span(span);
739+
vis.visit_span(span2);
738740
}
739741

740742
// No `noop_` prefix because there isn't a corresponding method in `MutVisitor`.

‎compiler/rustc_ast/src/token.rs

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -248,6 +248,7 @@ rustc_data_structures::static_assert_size!(TokenKind, 16);
248248
pub struct Token {
249249
pub kind: TokenKind,
250250
pub span: Span,
251+
pub span2: Span,
251252
}
252253

253254
impl TokenKind {
@@ -306,7 +307,7 @@ impl TokenKind {
306307

307308
impl Token {
308309
pub fn new(kind: TokenKind, span: Span) -> Self {
309-
Token { kind, span }
310+
Token { kind, span, span2: span }
310311
}
311312

312313
/// Some token that will be thrown away later.

‎compiler/rustc_expand/src/mbe/macro_parser.rs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -95,7 +95,7 @@ type NamedMatchVec = SmallVec<[NamedMatch; 1]>;
9595

9696
// This type is used a lot. Make sure it doesn't unintentionally get bigger.
9797
#[cfg(all(target_arch = "x86_64", target_pointer_width = "64"))]
98-
rustc_data_structures::static_assert_size!(NamedMatchVec, 48);
98+
rustc_data_structures::static_assert_size!(NamedMatchVec, 56);
9999

100100
#[derive(Clone)]
101101
enum MatcherKind<'tt> {

‎compiler/rustc_expand/src/mbe/quoted.rs

Lines changed: 9 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -57,7 +57,9 @@ pub(super) fn parse(
5757
match tree {
5858
TokenTree::MetaVar(start_sp, ident) if parsing_patterns => {
5959
let span = match trees.next() {
60-
Some(tokenstream::TokenTree::Token(Token { kind: token::Colon, span })) => {
60+
Some(tokenstream::TokenTree::Token(Token {
61+
kind: token::Colon, span, ..
62+
})) => {
6163
match trees.next() {
6264
Some(tokenstream::TokenTree::Token(token)) => match token.ident() {
6365
Some((frag, _)) => {
@@ -149,7 +151,7 @@ fn parse_tree(
149151
// Depending on what `tree` is, we could be parsing different parts of a macro
150152
match tree {
151153
// `tree` is a `$` token. Look at the next token in `trees`
152-
tokenstream::TokenTree::Token(Token { kind: token::Dollar, span }) => {
154+
tokenstream::TokenTree::Token(Token { kind: token::Dollar, span, .. }) => {
153155
// FIXME: Handle `None`-delimited groups in a more systematic way
154156
// during parsing.
155157
let mut next = outer_trees.next();
@@ -168,7 +170,7 @@ fn parse_tree(
168170
if delim != token::Paren {
169171
span_dollar_dollar_or_metavar_in_the_lhs_err(
170172
sess,
171-
&Token { kind: token::OpenDelim(delim), span: delim_span.entire() },
173+
&Token::new(token::OpenDelim(delim), delim_span.entire()),
172174
);
173175
}
174176
} else {
@@ -237,11 +239,13 @@ fn parse_tree(
237239
}
238240

239241
// `tree` is followed by another `$`. This is an escaped `$`.
240-
Some(tokenstream::TokenTree::Token(Token { kind: token::Dollar, span })) => {
242+
Some(tokenstream::TokenTree::Token(Token {
243+
kind: token::Dollar, span, ..
244+
})) => {
241245
if parsing_patterns {
242246
span_dollar_dollar_or_metavar_in_the_lhs_err(
243247
sess,
244-
&Token { kind: token::Dollar, span },
248+
&Token::new(token::Dollar, span),
245249
);
246250
} else {
247251
maybe_emit_macro_metavar_expr_feature(features, sess, span);

‎compiler/rustc_expand/src/proc_macro_server.rs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -59,7 +59,7 @@ impl FromInternal<(TreeAndSpacing, &'_ mut Vec<Self>, &mut Rustc<'_, '_>)>
5959
use rustc_ast::token::*;
6060

6161
let joint = spacing == Joint;
62-
let Token { kind, span } = match tree {
62+
let Token { kind, span, .. } = match tree {
6363
tokenstream::TokenTree::Delimited(span, delim, tts) => {
6464
let delimiter = Delimiter::from_internal(delim);
6565
return TokenTree::Group(Group { delimiter, stream: tts, span, flatten: false });

‎compiler/rustc_parse/src/parser/attr_wrapper.rs

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -96,7 +96,7 @@ struct LazyTokenStreamImpl {
9696
}
9797

9898
#[cfg(all(target_arch = "x86_64", target_pointer_width = "64"))]
99-
rustc_data_structures::static_assert_size!(LazyTokenStreamImpl, 144);
99+
rustc_data_structures::static_assert_size!(LazyTokenStreamImpl, 152);
100100

101101
impl CreateTokenStream for LazyTokenStreamImpl {
102102
fn create_token_stream(&self) -> AttrAnnotatedTokenStream {
@@ -414,10 +414,10 @@ fn make_token_stream(
414414
let mut token_and_spacing = iter.next();
415415
while let Some((token, spacing)) = token_and_spacing {
416416
match token {
417-
FlatToken::Token(Token { kind: TokenKind::OpenDelim(delim), span }) => {
417+
FlatToken::Token(Token { kind: TokenKind::OpenDelim(delim), span, .. }) => {
418418
stack.push(FrameData { open: span, open_delim: delim, inner: vec![] });
419419
}
420-
FlatToken::Token(Token { kind: TokenKind::CloseDelim(delim), span }) => {
420+
FlatToken::Token(Token { kind: TokenKind::CloseDelim(delim), span, .. }) => {
421421
// HACK: If we encounter a mismatched `None` delimiter at the top
422422
// level, just ignore it.
423423
if matches!(delim, DelimToken::NoDelim)

‎compiler/rustc_parse/src/parser/mod.rs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -299,7 +299,7 @@ impl TokenCursor {
299299
#[inline(always)]
300300
fn inlined_next_desugared(&mut self) -> (Token, Spacing) {
301301
let (data, attr_style, sp) = match self.inlined_next() {
302-
(Token { kind: token::DocComment(_, attr_style, data), span }, _) => {
302+
(Token { kind: token::DocComment(_, attr_style, data), span, .. }, _) => {
303303
(data, attr_style, span)
304304
}
305305
tok => return tok,

‎src/tools/rustfmt/src/macros.rs

Lines changed: 8 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -683,10 +683,9 @@ struct MacroArgParser {
683683
fn last_tok(tt: &TokenTree) -> Token {
684684
match *tt {
685685
TokenTree::Token(ref t) => t.clone(),
686-
TokenTree::Delimited(delim_span, delim, _) => Token {
687-
kind: TokenKind::CloseDelim(delim),
688-
span: delim_span.close,
689-
},
686+
TokenTree::Delimited(delim_span, delim, _) => {
687+
Token::new(TokenKind::CloseDelim(delim), delim_span.close)
688+
}
690689
}
691690
}
692691

@@ -695,14 +694,8 @@ impl MacroArgParser {
695694
MacroArgParser {
696695
buf: String::new(),
697696
is_meta_var: false,
698-
last_tok: Token {
699-
kind: TokenKind::Eof,
700-
span: DUMMY_SP,
701-
},
702-
start_tok: Token {
703-
kind: TokenKind::Eof,
704-
span: DUMMY_SP,
705-
},
697+
last_tok: Token::new(TokenKind::Eof, DUMMY_SP),
698+
start_tok: Token::new(TokenKind::Eof, DUMMY_SP),
706699
result: vec![],
707700
}
708701
}
@@ -862,6 +855,7 @@ impl MacroArgParser {
862855
TokenTree::Token(Token {
863856
kind: TokenKind::Dollar,
864857
span,
858+
..
865859
}) => {
866860
// We always want to add a separator before meta variables.
867861
if !self.buf.is_empty() {
@@ -870,10 +864,7 @@ impl MacroArgParser {
870864

871865
// Start keeping the name of this metavariable in the buffer.
872866
self.is_meta_var = true;
873-
self.start_tok = Token {
874-
kind: TokenKind::Dollar,
875-
span,
876-
};
867+
self.start_tok = Token::new(TokenKind::Dollar, span);
877868
}
878869
TokenTree::Token(Token {
879870
kind: TokenKind::Colon,
@@ -1150,6 +1141,7 @@ impl MacroParser {
11501141
if let Some(TokenTree::Token(Token {
11511142
kind: TokenKind::Semi,
11521143
span,
1144+
..
11531145
})) = self.toks.look_ahead(0)
11541146
{
11551147
hi = span.hi();

0 commit comments

Comments
 (0)