diff --git a/compiler/rustc_parse/src/parser/attr.rs b/compiler/rustc_parse/src/parser/attr.rs index 1b26fb3337043..df581467c363c 100644 --- a/compiler/rustc_parse/src/parser/attr.rs +++ b/compiler/rustc_parse/src/parser/attr.rs @@ -4,7 +4,7 @@ use rustc_ast::attr; use rustc_ast::token::{self, Nonterminal}; use rustc_ast_pretty::pprust; use rustc_errors::{error_code, PResult}; -use rustc_span::{sym, Span}; +use rustc_span::Span; use tracing::debug; @@ -304,15 +304,3 @@ impl<'a> Parser<'a> { Err(self.struct_span_err(self.token.span, &msg)) } } - -pub fn maybe_needs_tokens(attrs: &[ast::Attribute]) -> bool { - // One of the attributes may either itself be a macro, or apply derive macros (`derive`), - // or expand to macro attributes (`cfg_attr`). - attrs.iter().any(|attr| { - attr.ident().map_or(true, |ident| { - ident.name == sym::derive - || ident.name == sym::cfg_attr - || !rustc_feature::is_builtin_attr_name(ident.name) - }) - }) -} diff --git a/compiler/rustc_parse/src/parser/expr.rs b/compiler/rustc_parse/src/parser/expr.rs index 6db415ead415c..e417407b34dbe 100644 --- a/compiler/rustc_parse/src/parser/expr.rs +++ b/compiler/rustc_parse/src/parser/expr.rs @@ -472,7 +472,7 @@ impl<'a> Parser<'a> { /// Parses a prefix-unary-operator expr. fn parse_prefix_expr(&mut self, attrs: Option) -> PResult<'a, P> { let attrs = self.parse_or_use_outer_attributes(attrs)?; - let needs_tokens = super::attr::maybe_needs_tokens(&attrs); + let needs_tokens = self.maybe_needs_tokens(&attrs); let do_parse = |this: &mut Parser<'a>| { let lo = this.token.span; // Note: when adding new unary operators, don't forget to adjust TokenKind::can_begin_expr() diff --git a/compiler/rustc_parse/src/parser/item.rs b/compiler/rustc_parse/src/parser/item.rs index 810ae61307c19..8c388e6d7f75e 100644 --- a/compiler/rustc_parse/src/parser/item.rs +++ b/compiler/rustc_parse/src/parser/item.rs @@ -116,7 +116,7 @@ impl<'a> Parser<'a> { Some(item.into_inner()) }); - let needs_tokens = super::attr::maybe_needs_tokens(&attrs); + let needs_tokens = self.maybe_needs_tokens(&attrs); let mut unclosed_delims = vec![]; let parse_item = |this: &mut Self| { diff --git a/compiler/rustc_parse/src/parser/mod.rs b/compiler/rustc_parse/src/parser/mod.rs index 5d7ea5b8d578e..ee5cd18da6ffb 100644 --- a/compiler/rustc_parse/src/parser/mod.rs +++ b/compiler/rustc_parse/src/parser/mod.rs @@ -116,6 +116,7 @@ pub struct Parser<'a> { pub last_type_ascription: Option<(Span, bool /* likely path typo */)>, /// If present, this `Parser` is not parsing Rust code but rather a macro call. subparser_name: Option<&'static str>, + collecting: bool, } impl<'a> Drop for Parser<'a> { @@ -367,6 +368,7 @@ impl<'a> Parser<'a> { last_unexpected_token_span: None, last_type_ascription: None, subparser_name, + collecting: false, }; // Make parser point to the first token. @@ -1218,6 +1220,31 @@ impl<'a> Parser<'a> { } } + /// Returns `true` if we might need to have tokens available for + /// the attribute target, and `false` if we definitely do not need tokens. + pub(super) fn maybe_needs_tokens(&self, attrs: &[ast::Attribute]) -> bool { + // If we are already inside a call to `collect_tokens`, then always + // collect tokens. If the outer `collect_tokens` call ends up returning + // the same AST node (for example, during nonterminal parsing), we + // want to make sure that we already have a `LazyTokenStream` set + // (from the inner `collect_tokens` call). The innermost call + // will correctly exclude the attributes from the `TokenStream`, + // while the outer call will end up capturing the attribute tokens + // in its (discarded) `LazyTokenStream` + self.collecting || { + // One of the attributes may either itself be a macro, + // or expand to macro attributes (`cfg_attr`). In either case, + // we need tokens available to pass to the macro invocation + attrs.iter().any(|attr| { + attr.ident().map_or(true, |ident| { + ident.name == sym::derive + || ident.name == sym::cfg_attr + || !rustc_feature::is_builtin_attr_name(ident.name) + }) + }) + } + } + /// Records all tokens consumed by the provided callback, /// including the current token. These tokens are collected /// into a `LazyTokenStream`, and returned along with the result @@ -1249,7 +1276,10 @@ impl<'a> Parser<'a> { append_unglued_token: self.token_cursor.append_unglued_token.clone(), }; - let mut ret = f(self)?; + let prev_collecting = std::mem::replace(&mut self.collecting, true); + let ret = f(self); + self.collecting = prev_collecting; + let mut ret = ret?; // Produces a `TokenStream` on-demand. Using `cursor_snapshot` // and `num_calls`, we can reconstruct the `TokenStream` seen diff --git a/src/test/ui/ast-json/ast-json-noexpand-output.stdout b/src/test/ui/ast-json/ast-json-noexpand-output.stdout index 9c29a5f2337cd..02342af8dc53e 100644 --- a/src/test/ui/ast-json/ast-json-noexpand-output.stdout +++ b/src/test/ui/ast-json/ast-json-noexpand-output.stdout @@ -1 +1 @@ -{"module":{"inner":{"lo":0,"hi":0},"unsafety":"No","items":[{"attrs":[],"id":0,"span":{"lo":0,"hi":0},"vis":{"kind":"Inherited","span":{"lo":0,"hi":0},"tokens":null},"ident":{"name":"core","span":{"lo":0,"hi":0}},"kind":{"variant":"ExternCrate","fields":[null]},"tokens":null}],"inline":true},"attrs":[{"kind":{"variant":"Normal","fields":[{"path":{"span":{"lo":0,"hi":0},"segments":[{"ident":{"name":"crate_type","span":{"lo":0,"hi":0}},"id":0,"args":null}],"tokens":null},"args":{"variant":"Eq","fields":[{"lo":0,"hi":0},{"kind":{"variant":"Interpolated","fields":[{"variant":"NtExpr","fields":[{"id":0,"kind":{"variant":"Lit","fields":[{"token":{"kind":"Str","symbol":"lib","suffix":null},"kind":{"variant":"Str","fields":["lib","Cooked"]},"span":{"lo":0,"hi":0}}]},"span":{"lo":0,"hi":0},"attrs":{"0":null},"tokens":null}]}]},"span":{"lo":0,"hi":0}}]},"tokens":null},{"0":[[{"variant":"Token","fields":[{"kind":"Pound","span":{"lo":0,"hi":0}}]},"Joint"],[{"variant":"Token","fields":[{"kind":"Not","span":{"lo":0,"hi":0}}]},"Alone"],[{"variant":"Delimited","fields":[{"open":{"lo":0,"hi":0},"close":{"lo":0,"hi":0}},"Bracket",{"0":[[{"variant":"Token","fields":[{"kind":{"variant":"Ident","fields":["crate_type",false]},"span":{"lo":0,"hi":0}}]},"Alone"],[{"variant":"Token","fields":[{"kind":"Eq","span":{"lo":0,"hi":0}}]},"Alone"],[{"variant":"Token","fields":[{"kind":{"variant":"Literal","fields":[{"kind":"Str","symbol":"lib","suffix":null}]},"span":{"lo":0,"hi":0}}]},"Alone"]]}]},"Alone"]]}]},"id":null,"style":"Inner","span":{"lo":0,"hi":0}}],"span":{"lo":0,"hi":0},"proc_macros":[]} +{"module":{"inner":{"lo":0,"hi":0},"unsafety":"No","items":[{"attrs":[],"id":0,"span":{"lo":0,"hi":0},"vis":{"kind":"Inherited","span":{"lo":0,"hi":0},"tokens":null},"ident":{"name":"core","span":{"lo":0,"hi":0}},"kind":{"variant":"ExternCrate","fields":[null]},"tokens":null}],"inline":true},"attrs":[{"kind":{"variant":"Normal","fields":[{"path":{"span":{"lo":0,"hi":0},"segments":[{"ident":{"name":"crate_type","span":{"lo":0,"hi":0}},"id":0,"args":null}],"tokens":null},"args":{"variant":"Eq","fields":[{"lo":0,"hi":0},{"kind":{"variant":"Interpolated","fields":[{"variant":"NtExpr","fields":[{"id":0,"kind":{"variant":"Lit","fields":[{"token":{"kind":"Str","symbol":"lib","suffix":null},"kind":{"variant":"Str","fields":["lib","Cooked"]},"span":{"lo":0,"hi":0}}]},"span":{"lo":0,"hi":0},"attrs":{"0":null},"tokens":{"0":[[{"variant":"Token","fields":[{"kind":{"variant":"Literal","fields":[{"kind":"Str","symbol":"lib","suffix":null}]},"span":{"lo":0,"hi":0}}]},"Alone"]]}}]}]},"span":{"lo":0,"hi":0}}]},"tokens":null},{"0":[[{"variant":"Token","fields":[{"kind":"Pound","span":{"lo":0,"hi":0}}]},"Joint"],[{"variant":"Token","fields":[{"kind":"Not","span":{"lo":0,"hi":0}}]},"Alone"],[{"variant":"Delimited","fields":[{"open":{"lo":0,"hi":0},"close":{"lo":0,"hi":0}},"Bracket",{"0":[[{"variant":"Token","fields":[{"kind":{"variant":"Ident","fields":["crate_type",false]},"span":{"lo":0,"hi":0}}]},"Alone"],[{"variant":"Token","fields":[{"kind":"Eq","span":{"lo":0,"hi":0}}]},"Alone"],[{"variant":"Token","fields":[{"kind":{"variant":"Literal","fields":[{"kind":"Str","symbol":"lib","suffix":null}]},"span":{"lo":0,"hi":0}}]},"Alone"]]}]},"Alone"]]}]},"id":null,"style":"Inner","span":{"lo":0,"hi":0}}],"span":{"lo":0,"hi":0},"proc_macros":[]} diff --git a/src/test/ui/ast-json/ast-json-output.stdout b/src/test/ui/ast-json/ast-json-output.stdout index cccd51985dc83..235f39c567b8e 100644 --- a/src/test/ui/ast-json/ast-json-output.stdout +++ b/src/test/ui/ast-json/ast-json-output.stdout @@ -1 +1 @@ -{"module":{"inner":{"lo":0,"hi":0},"unsafety":"No","items":[{"attrs":[{"kind":{"variant":"Normal","fields":[{"path":{"span":{"lo":0,"hi":0},"segments":[{"ident":{"name":"prelude_import","span":{"lo":0,"hi":0}},"id":0,"args":null}],"tokens":null},"args":"Empty","tokens":null},null]},"id":null,"style":"Outer","span":{"lo":0,"hi":0}}],"id":0,"span":{"lo":0,"hi":0},"vis":{"kind":"Inherited","span":{"lo":0,"hi":0},"tokens":null},"ident":{"name":"","span":{"lo":0,"hi":0}},"kind":{"variant":"Use","fields":[{"prefix":{"span":{"lo":0,"hi":0},"segments":[{"ident":{"name":"{{root}}","span":{"lo":0,"hi":0}},"id":0,"args":null},{"ident":{"name":"std","span":{"lo":0,"hi":0}},"id":0,"args":null},{"ident":{"name":"prelude","span":{"lo":0,"hi":0}},"id":0,"args":null},{"ident":{"name":"v1","span":{"lo":0,"hi":0}},"id":0,"args":null}],"tokens":null},"kind":"Glob","span":{"lo":0,"hi":0}}]},"tokens":null},{"attrs":[{"kind":{"variant":"Normal","fields":[{"path":{"span":{"lo":0,"hi":0},"segments":[{"ident":{"name":"macro_use","span":{"lo":0,"hi":0}},"id":0,"args":null}],"tokens":null},"args":"Empty","tokens":null},null]},"id":null,"style":"Outer","span":{"lo":0,"hi":0}}],"id":0,"span":{"lo":0,"hi":0},"vis":{"kind":"Inherited","span":{"lo":0,"hi":0},"tokens":null},"ident":{"name":"std","span":{"lo":0,"hi":0}},"kind":{"variant":"ExternCrate","fields":[null]},"tokens":null},{"attrs":[],"id":0,"span":{"lo":0,"hi":0},"vis":{"kind":"Inherited","span":{"lo":0,"hi":0},"tokens":null},"ident":{"name":"core","span":{"lo":0,"hi":0}},"kind":{"variant":"ExternCrate","fields":[null]},"tokens":null}],"inline":true},"attrs":[{"kind":{"variant":"Normal","fields":[{"path":{"span":{"lo":0,"hi":0},"segments":[{"ident":{"name":"crate_type","span":{"lo":0,"hi":0}},"id":0,"args":null}],"tokens":null},"args":{"variant":"Eq","fields":[{"lo":0,"hi":0},{"kind":{"variant":"Interpolated","fields":[{"variant":"NtExpr","fields":[{"id":0,"kind":{"variant":"Lit","fields":[{"token":{"kind":"Str","symbol":"lib","suffix":null},"kind":{"variant":"Str","fields":["lib","Cooked"]},"span":{"lo":0,"hi":0}}]},"span":{"lo":0,"hi":0},"attrs":{"0":null},"tokens":null}]}]},"span":{"lo":0,"hi":0}}]},"tokens":null},{"0":[[{"variant":"Token","fields":[{"kind":"Pound","span":{"lo":0,"hi":0}}]},"Joint"],[{"variant":"Token","fields":[{"kind":"Not","span":{"lo":0,"hi":0}}]},"Alone"],[{"variant":"Delimited","fields":[{"open":{"lo":0,"hi":0},"close":{"lo":0,"hi":0}},"Bracket",{"0":[[{"variant":"Token","fields":[{"kind":{"variant":"Ident","fields":["crate_type",false]},"span":{"lo":0,"hi":0}}]},"Alone"],[{"variant":"Token","fields":[{"kind":"Eq","span":{"lo":0,"hi":0}}]},"Alone"],[{"variant":"Token","fields":[{"kind":{"variant":"Literal","fields":[{"kind":"Str","symbol":"lib","suffix":null}]},"span":{"lo":0,"hi":0}}]},"Alone"]]}]},"Alone"]]}]},"id":null,"style":"Inner","span":{"lo":0,"hi":0}}],"span":{"lo":0,"hi":0},"proc_macros":[]} +{"module":{"inner":{"lo":0,"hi":0},"unsafety":"No","items":[{"attrs":[{"kind":{"variant":"Normal","fields":[{"path":{"span":{"lo":0,"hi":0},"segments":[{"ident":{"name":"prelude_import","span":{"lo":0,"hi":0}},"id":0,"args":null}],"tokens":null},"args":"Empty","tokens":null},null]},"id":null,"style":"Outer","span":{"lo":0,"hi":0}}],"id":0,"span":{"lo":0,"hi":0},"vis":{"kind":"Inherited","span":{"lo":0,"hi":0},"tokens":null},"ident":{"name":"","span":{"lo":0,"hi":0}},"kind":{"variant":"Use","fields":[{"prefix":{"span":{"lo":0,"hi":0},"segments":[{"ident":{"name":"{{root}}","span":{"lo":0,"hi":0}},"id":0,"args":null},{"ident":{"name":"std","span":{"lo":0,"hi":0}},"id":0,"args":null},{"ident":{"name":"prelude","span":{"lo":0,"hi":0}},"id":0,"args":null},{"ident":{"name":"v1","span":{"lo":0,"hi":0}},"id":0,"args":null}],"tokens":null},"kind":"Glob","span":{"lo":0,"hi":0}}]},"tokens":null},{"attrs":[{"kind":{"variant":"Normal","fields":[{"path":{"span":{"lo":0,"hi":0},"segments":[{"ident":{"name":"macro_use","span":{"lo":0,"hi":0}},"id":0,"args":null}],"tokens":null},"args":"Empty","tokens":null},null]},"id":null,"style":"Outer","span":{"lo":0,"hi":0}}],"id":0,"span":{"lo":0,"hi":0},"vis":{"kind":"Inherited","span":{"lo":0,"hi":0},"tokens":null},"ident":{"name":"std","span":{"lo":0,"hi":0}},"kind":{"variant":"ExternCrate","fields":[null]},"tokens":null},{"attrs":[],"id":0,"span":{"lo":0,"hi":0},"vis":{"kind":"Inherited","span":{"lo":0,"hi":0},"tokens":null},"ident":{"name":"core","span":{"lo":0,"hi":0}},"kind":{"variant":"ExternCrate","fields":[null]},"tokens":null}],"inline":true},"attrs":[{"kind":{"variant":"Normal","fields":[{"path":{"span":{"lo":0,"hi":0},"segments":[{"ident":{"name":"crate_type","span":{"lo":0,"hi":0}},"id":0,"args":null}],"tokens":null},"args":{"variant":"Eq","fields":[{"lo":0,"hi":0},{"kind":{"variant":"Interpolated","fields":[{"variant":"NtExpr","fields":[{"id":0,"kind":{"variant":"Lit","fields":[{"token":{"kind":"Str","symbol":"lib","suffix":null},"kind":{"variant":"Str","fields":["lib","Cooked"]},"span":{"lo":0,"hi":0}}]},"span":{"lo":0,"hi":0},"attrs":{"0":null},"tokens":{"0":[[{"variant":"Token","fields":[{"kind":{"variant":"Literal","fields":[{"kind":"Str","symbol":"lib","suffix":null}]},"span":{"lo":0,"hi":0}}]},"Alone"]]}}]}]},"span":{"lo":0,"hi":0}}]},"tokens":null},{"0":[[{"variant":"Token","fields":[{"kind":"Pound","span":{"lo":0,"hi":0}}]},"Joint"],[{"variant":"Token","fields":[{"kind":"Not","span":{"lo":0,"hi":0}}]},"Alone"],[{"variant":"Delimited","fields":[{"open":{"lo":0,"hi":0},"close":{"lo":0,"hi":0}},"Bracket",{"0":[[{"variant":"Token","fields":[{"kind":{"variant":"Ident","fields":["crate_type",false]},"span":{"lo":0,"hi":0}}]},"Alone"],[{"variant":"Token","fields":[{"kind":"Eq","span":{"lo":0,"hi":0}}]},"Alone"],[{"variant":"Token","fields":[{"kind":{"variant":"Literal","fields":[{"kind":"Str","symbol":"lib","suffix":null}]},"span":{"lo":0,"hi":0}}]},"Alone"]]}]},"Alone"]]}]},"id":null,"style":"Inner","span":{"lo":0,"hi":0}}],"span":{"lo":0,"hi":0},"proc_macros":[]} diff --git a/src/test/ui/proc-macro/issue-81007-item-attrs.rs b/src/test/ui/proc-macro/issue-81007-item-attrs.rs new file mode 100644 index 0000000000000..ea27d54ee4148 --- /dev/null +++ b/src/test/ui/proc-macro/issue-81007-item-attrs.rs @@ -0,0 +1,31 @@ +// check-pass +// edition:2018 +// compile-flags: -Z span-debug +// aux-build:test-macros.rs + +#![feature(rustc_attrs)] + +#![no_std] // Don't load unnecessary hygiene information from std +extern crate std; + +#[macro_use] extern crate test_macros; + +macro_rules! capture_item { + ($item:item) => { + #[print_attr] + $item + } +} + +capture_item! { + /// A doc comment + struct Foo {} +} + +capture_item! { + #[rustc_dummy] + /// Another comment comment + struct Bar {} +} + +fn main() {} diff --git a/src/test/ui/proc-macro/issue-81007-item-attrs.stdout b/src/test/ui/proc-macro/issue-81007-item-attrs.stdout new file mode 100644 index 0000000000000..6f880a1202170 --- /dev/null +++ b/src/test/ui/proc-macro/issue-81007-item-attrs.stdout @@ -0,0 +1,99 @@ +PRINT-ATTR INPUT (DISPLAY): #[doc = r" A doc comment"] struct Foo { } +PRINT-ATTR INPUT (DEBUG): TokenStream [ + Punct { + ch: '#', + spacing: Alone, + span: $DIR/issue-81007-item-attrs.rs:21:5: 21:22 (#0), + }, + Group { + delimiter: Bracket, + stream: TokenStream [ + Ident { + ident: "doc", + span: $DIR/issue-81007-item-attrs.rs:21:5: 21:22 (#0), + }, + Punct { + ch: '=', + spacing: Alone, + span: $DIR/issue-81007-item-attrs.rs:21:5: 21:22 (#0), + }, + Literal { + kind: StrRaw(0), + symbol: " A doc comment", + suffix: None, + span: $DIR/issue-81007-item-attrs.rs:21:5: 21:22 (#0), + }, + ], + span: $DIR/issue-81007-item-attrs.rs:21:5: 21:22 (#0), + }, + Ident { + ident: "struct", + span: $DIR/issue-81007-item-attrs.rs:22:5: 22:11 (#0), + }, + Ident { + ident: "Foo", + span: $DIR/issue-81007-item-attrs.rs:22:12: 22:15 (#0), + }, + Group { + delimiter: Brace, + stream: TokenStream [], + span: $DIR/issue-81007-item-attrs.rs:22:16: 22:18 (#0), + }, +] +PRINT-ATTR INPUT (DISPLAY): #[rustc_dummy] #[doc = r" Another comment comment"] struct Bar { } +PRINT-ATTR INPUT (DEBUG): TokenStream [ + Punct { + ch: '#', + spacing: Alone, + span: $DIR/issue-81007-item-attrs.rs:26:5: 26:6 (#0), + }, + Group { + delimiter: Bracket, + stream: TokenStream [ + Ident { + ident: "rustc_dummy", + span: $DIR/issue-81007-item-attrs.rs:26:7: 26:18 (#0), + }, + ], + span: $DIR/issue-81007-item-attrs.rs:26:6: 26:19 (#0), + }, + Punct { + ch: '#', + spacing: Alone, + span: $DIR/issue-81007-item-attrs.rs:27:5: 27:32 (#0), + }, + Group { + delimiter: Bracket, + stream: TokenStream [ + Ident { + ident: "doc", + span: $DIR/issue-81007-item-attrs.rs:27:5: 27:32 (#0), + }, + Punct { + ch: '=', + spacing: Alone, + span: $DIR/issue-81007-item-attrs.rs:27:5: 27:32 (#0), + }, + Literal { + kind: StrRaw(0), + symbol: " Another comment comment", + suffix: None, + span: $DIR/issue-81007-item-attrs.rs:27:5: 27:32 (#0), + }, + ], + span: $DIR/issue-81007-item-attrs.rs:27:5: 27:32 (#0), + }, + Ident { + ident: "struct", + span: $DIR/issue-81007-item-attrs.rs:28:5: 28:11 (#0), + }, + Ident { + ident: "Bar", + span: $DIR/issue-81007-item-attrs.rs:28:12: 28:15 (#0), + }, + Group { + delimiter: Brace, + stream: TokenStream [], + span: $DIR/issue-81007-item-attrs.rs:28:16: 28:18 (#0), + }, +]