diff --git a/compiler/rustc_ast/src/attr/mod.rs b/compiler/rustc_ast/src/attr/mod.rs index 550c66e3d3b20..9b25a73575b7b 100644 --- a/compiler/rustc_ast/src/attr/mod.rs +++ b/compiler/rustc_ast/src/attr/mod.rs @@ -407,6 +407,7 @@ impl MetaItem { Some(TokenTree::Token(Token { kind: kind @ (token::Ident(..) | token::ModSep), span, + .. })) => 'arm: { let mut segments = if let token::Ident(name, _) = kind { if let Some(TokenTree::Token(Token { kind: token::ModSep, .. })) = tokens.peek() @@ -420,8 +421,9 @@ impl MetaItem { vec![PathSegment::path_root(span)] }; loop { - if let Some(TokenTree::Token(Token { kind: token::Ident(name, _), span })) = - tokens.next().map(TokenTree::uninterpolate) + if let Some(TokenTree::Token(Token { + kind: token::Ident(name, _), span, .. + })) = tokens.next().map(TokenTree::uninterpolate) { segments.push(PathSegment::from_ident(Ident::new(name, span))); } else { diff --git a/compiler/rustc_ast/src/mut_visit.rs b/compiler/rustc_ast/src/mut_visit.rs index 32621eb5f2f66..a733776022706 100644 --- a/compiler/rustc_ast/src/mut_visit.rs +++ b/compiler/rustc_ast/src/mut_visit.rs @@ -719,13 +719,14 @@ pub fn visit_lazy_tts(lazy_tts: &mut Option, vis // In practice the ident part is not actually used by specific visitors right now, // but there's a test below checking that it works. pub fn visit_token(t: &mut Token, vis: &mut T) { - let Token { kind, span } = t; + let Token { kind, span, span2 } = t; match kind { token::Ident(name, _) | token::Lifetime(name) => { let mut ident = Ident::new(*name, *span); vis.visit_ident(&mut ident); *name = ident.name; *span = ident.span; + vis.visit_span(span2); return; // Avoid visiting the span for the second time. } token::Interpolated(nt) => { @@ -735,6 +736,7 @@ pub fn visit_token(t: &mut Token, vis: &mut T) { _ => {} } vis.visit_span(span); + vis.visit_span(span2); } // No `noop_` prefix because there isn't a corresponding method in `MutVisitor`. diff --git a/compiler/rustc_ast/src/token.rs b/compiler/rustc_ast/src/token.rs index 2132cdfc001b6..4f9b49ccd70f8 100644 --- a/compiler/rustc_ast/src/token.rs +++ b/compiler/rustc_ast/src/token.rs @@ -248,6 +248,7 @@ rustc_data_structures::static_assert_size!(TokenKind, 16); pub struct Token { pub kind: TokenKind, pub span: Span, + pub span2: Span, } impl TokenKind { @@ -306,7 +307,7 @@ impl TokenKind { impl Token { pub fn new(kind: TokenKind, span: Span) -> Self { - Token { kind, span } + Token { kind, span, span2: span } } /// Some token that will be thrown away later. diff --git a/compiler/rustc_expand/src/mbe/macro_parser.rs b/compiler/rustc_expand/src/mbe/macro_parser.rs index 7d4de72c3fef6..231e2f66e2c7f 100644 --- a/compiler/rustc_expand/src/mbe/macro_parser.rs +++ b/compiler/rustc_expand/src/mbe/macro_parser.rs @@ -95,7 +95,7 @@ type NamedMatchVec = SmallVec<[NamedMatch; 1]>; // This type is used a lot. Make sure it doesn't unintentionally get bigger. #[cfg(all(target_arch = "x86_64", target_pointer_width = "64"))] -rustc_data_structures::static_assert_size!(NamedMatchVec, 48); +rustc_data_structures::static_assert_size!(NamedMatchVec, 56); #[derive(Clone)] enum MatcherKind<'tt> { diff --git a/compiler/rustc_expand/src/mbe/quoted.rs b/compiler/rustc_expand/src/mbe/quoted.rs index 48abbd7c18e14..4da3e79e7579d 100644 --- a/compiler/rustc_expand/src/mbe/quoted.rs +++ b/compiler/rustc_expand/src/mbe/quoted.rs @@ -57,7 +57,9 @@ pub(super) fn parse( match tree { TokenTree::MetaVar(start_sp, ident) if parsing_patterns => { let span = match trees.next() { - Some(tokenstream::TokenTree::Token(Token { kind: token::Colon, span })) => { + Some(tokenstream::TokenTree::Token(Token { + kind: token::Colon, span, .. + })) => { match trees.next() { Some(tokenstream::TokenTree::Token(token)) => match token.ident() { Some((frag, _)) => { @@ -149,7 +151,7 @@ fn parse_tree( // Depending on what `tree` is, we could be parsing different parts of a macro match tree { // `tree` is a `$` token. Look at the next token in `trees` - tokenstream::TokenTree::Token(Token { kind: token::Dollar, span }) => { + tokenstream::TokenTree::Token(Token { kind: token::Dollar, span, .. }) => { // FIXME: Handle `None`-delimited groups in a more systematic way // during parsing. let mut next = outer_trees.next(); @@ -168,7 +170,7 @@ fn parse_tree( if delim != token::Paren { span_dollar_dollar_or_metavar_in_the_lhs_err( sess, - &Token { kind: token::OpenDelim(delim), span: delim_span.entire() }, + &Token::new(token::OpenDelim(delim), delim_span.entire()), ); } } else { @@ -237,11 +239,13 @@ fn parse_tree( } // `tree` is followed by another `$`. This is an escaped `$`. - Some(tokenstream::TokenTree::Token(Token { kind: token::Dollar, span })) => { + Some(tokenstream::TokenTree::Token(Token { + kind: token::Dollar, span, .. + })) => { if parsing_patterns { span_dollar_dollar_or_metavar_in_the_lhs_err( sess, - &Token { kind: token::Dollar, span }, + &Token::new(token::Dollar, span), ); } else { maybe_emit_macro_metavar_expr_feature(features, sess, span); diff --git a/compiler/rustc_expand/src/proc_macro_server.rs b/compiler/rustc_expand/src/proc_macro_server.rs index bfdf99762f57a..cad9918218947 100644 --- a/compiler/rustc_expand/src/proc_macro_server.rs +++ b/compiler/rustc_expand/src/proc_macro_server.rs @@ -59,7 +59,7 @@ impl FromInternal<(TreeAndSpacing, &'_ mut Vec, &mut Rustc<'_, '_>)> use rustc_ast::token::*; let joint = spacing == Joint; - let Token { kind, span } = match tree { + let Token { kind, span, .. } = match tree { tokenstream::TokenTree::Delimited(span, delim, tts) => { let delimiter = Delimiter::from_internal(delim); return TokenTree::Group(Group { delimiter, stream: tts, span, flatten: false }); diff --git a/compiler/rustc_parse/src/parser/attr_wrapper.rs b/compiler/rustc_parse/src/parser/attr_wrapper.rs index 5ee9c339bb7aa..da8bf9abc595f 100644 --- a/compiler/rustc_parse/src/parser/attr_wrapper.rs +++ b/compiler/rustc_parse/src/parser/attr_wrapper.rs @@ -96,7 +96,7 @@ struct LazyTokenStreamImpl { } #[cfg(all(target_arch = "x86_64", target_pointer_width = "64"))] -rustc_data_structures::static_assert_size!(LazyTokenStreamImpl, 144); +rustc_data_structures::static_assert_size!(LazyTokenStreamImpl, 152); impl CreateTokenStream for LazyTokenStreamImpl { fn create_token_stream(&self) -> AttrAnnotatedTokenStream { @@ -414,10 +414,10 @@ fn make_token_stream( let mut token_and_spacing = iter.next(); while let Some((token, spacing)) = token_and_spacing { match token { - FlatToken::Token(Token { kind: TokenKind::OpenDelim(delim), span }) => { + FlatToken::Token(Token { kind: TokenKind::OpenDelim(delim), span, .. }) => { stack.push(FrameData { open: span, open_delim: delim, inner: vec![] }); } - FlatToken::Token(Token { kind: TokenKind::CloseDelim(delim), span }) => { + FlatToken::Token(Token { kind: TokenKind::CloseDelim(delim), span, .. }) => { // HACK: If we encounter a mismatched `None` delimiter at the top // level, just ignore it. if matches!(delim, DelimToken::NoDelim) diff --git a/compiler/rustc_parse/src/parser/mod.rs b/compiler/rustc_parse/src/parser/mod.rs index b242c1e050dc0..26fe3f30f5250 100644 --- a/compiler/rustc_parse/src/parser/mod.rs +++ b/compiler/rustc_parse/src/parser/mod.rs @@ -299,7 +299,7 @@ impl TokenCursor { #[inline(always)] fn inlined_next_desugared(&mut self) -> (Token, Spacing) { let (data, attr_style, sp) = match self.inlined_next() { - (Token { kind: token::DocComment(_, attr_style, data), span }, _) => { + (Token { kind: token::DocComment(_, attr_style, data), span, .. }, _) => { (data, attr_style, span) } tok => return tok, diff --git a/src/tools/rustfmt/src/macros.rs b/src/tools/rustfmt/src/macros.rs index 664f152e8be1d..2752a4d96203f 100644 --- a/src/tools/rustfmt/src/macros.rs +++ b/src/tools/rustfmt/src/macros.rs @@ -683,10 +683,9 @@ struct MacroArgParser { fn last_tok(tt: &TokenTree) -> Token { match *tt { TokenTree::Token(ref t) => t.clone(), - TokenTree::Delimited(delim_span, delim, _) => Token { - kind: TokenKind::CloseDelim(delim), - span: delim_span.close, - }, + TokenTree::Delimited(delim_span, delim, _) => { + Token::new(TokenKind::CloseDelim(delim), delim_span.close) + } } } @@ -695,14 +694,8 @@ impl MacroArgParser { MacroArgParser { buf: String::new(), is_meta_var: false, - last_tok: Token { - kind: TokenKind::Eof, - span: DUMMY_SP, - }, - start_tok: Token { - kind: TokenKind::Eof, - span: DUMMY_SP, - }, + last_tok: Token::new(TokenKind::Eof, DUMMY_SP), + start_tok: Token::new(TokenKind::Eof, DUMMY_SP), result: vec![], } } @@ -862,6 +855,7 @@ impl MacroArgParser { TokenTree::Token(Token { kind: TokenKind::Dollar, span, + .. }) => { // We always want to add a separator before meta variables. if !self.buf.is_empty() { @@ -870,10 +864,7 @@ impl MacroArgParser { // Start keeping the name of this metavariable in the buffer. self.is_meta_var = true; - self.start_tok = Token { - kind: TokenKind::Dollar, - span, - }; + self.start_tok = Token::new(TokenKind::Dollar, span); } TokenTree::Token(Token { kind: TokenKind::Colon, @@ -1150,6 +1141,7 @@ impl MacroParser { if let Some(TokenTree::Token(Token { kind: TokenKind::Semi, span, + .. })) = self.toks.look_ahead(0) { hi = span.hi();