From 1a7591d53b2a6b035325470c13fa7f2b389d74ea Mon Sep 17 00:00:00 2001 From: Mara Bos Date: Tue, 20 Jul 2021 19:21:19 +0200 Subject: [PATCH 1/4] Error when proc macro derive output cannot be fully parsed. --- compiler/rustc_expand/src/proc_macro.rs | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/compiler/rustc_expand/src/proc_macro.rs b/compiler/rustc_expand/src/proc_macro.rs index 3f84979ac05e7..283d03bdc5a0f 100644 --- a/compiler/rustc_expand/src/proc_macro.rs +++ b/compiler/rustc_expand/src/proc_macro.rs @@ -133,8 +133,11 @@ impl MultiItemModifier for ProcMacroDerive { } } - // fail if there have been errors emitted - if ecx.sess.parse_sess.span_diagnostic.err_count() > error_count_before { + // fail if there have been errors emitted, + // or if there's any unparsed tokens left. + if parser.token != token::Eof + || ecx.sess.parse_sess.span_diagnostic.err_count() > error_count_before + { ecx.struct_span_err(span, "proc-macro derive produced unparseable tokens").emit(); } From 5e1bf5670e144d76cf9881ae32a14b72befaeff1 Mon Sep 17 00:00:00 2001 From: Mara Bos Date: Tue, 20 Jul 2021 19:21:43 +0200 Subject: [PATCH 2/4] Add test for incomplete derive macro output parse. --- src/test/ui/proc-macro/auxiliary/derive-bad.rs | 5 +++++ src/test/ui/proc-macro/derive-bad.rs | 4 ++++ src/test/ui/proc-macro/derive-bad.stderr | 8 +++++++- 3 files changed, 16 insertions(+), 1 deletion(-) diff --git a/src/test/ui/proc-macro/auxiliary/derive-bad.rs b/src/test/ui/proc-macro/auxiliary/derive-bad.rs index 90bb9b1baf2b7..4d260748b695e 100644 --- a/src/test/ui/proc-macro/auxiliary/derive-bad.rs +++ b/src/test/ui/proc-macro/auxiliary/derive-bad.rs @@ -11,3 +11,8 @@ use proc_macro::TokenStream; pub fn derive_a(_input: TokenStream) -> TokenStream { "struct A { inner }".parse().unwrap() } + +#[proc_macro_derive(B)] +pub fn derive_b(_input: TokenStream) -> TokenStream { + "const _: () = (); { HEY! }".parse().unwrap() +} diff --git a/src/test/ui/proc-macro/derive-bad.rs b/src/test/ui/proc-macro/derive-bad.rs index cb5188b5fb43f..8b097a40f704c 100644 --- a/src/test/ui/proc-macro/derive-bad.rs +++ b/src/test/ui/proc-macro/derive-bad.rs @@ -8,4 +8,8 @@ extern crate derive_bad; //~| ERROR expected `:`, found `}` struct A; //~ ERROR the name `A` is defined multiple times +#[derive(B)] +//~^ ERROR proc-macro derive produced unparseable tokens +struct B; + fn main() {} diff --git a/src/test/ui/proc-macro/derive-bad.stderr b/src/test/ui/proc-macro/derive-bad.stderr index ae48141fb3133..9cdf4e3ab2456 100644 --- a/src/test/ui/proc-macro/derive-bad.stderr +++ b/src/test/ui/proc-macro/derive-bad.stderr @@ -23,6 +23,12 @@ LL | struct A; | = note: `A` must be defined only once in the type namespace of this module -error: aborting due to 3 previous errors +error: proc-macro derive produced unparseable tokens + --> $DIR/derive-bad.rs:11:10 + | +LL | #[derive(B)] + | ^ + +error: aborting due to 4 previous errors For more information about this error, try `rustc --explain E0428`. From cc8e7692f5bb20442cef22076442ff1891cb698b Mon Sep 17 00:00:00 2001 From: Mara Bos Date: Tue, 20 Jul 2021 21:59:25 +0200 Subject: [PATCH 3/4] Show the unexpected token when parsing derive macro output. --- compiler/rustc_expand/src/proc_macro.rs | 20 +++++++++++++++----- compiler/rustc_parse/src/parser/mod.rs | 2 +- src/test/ui/proc-macro/derive-bad.rs | 1 + src/test/ui/proc-macro/derive-bad.stderr | 10 +++++++++- 4 files changed, 26 insertions(+), 7 deletions(-) diff --git a/compiler/rustc_expand/src/proc_macro.rs b/compiler/rustc_expand/src/proc_macro.rs index 283d03bdc5a0f..d52484d9288ca 100644 --- a/compiler/rustc_expand/src/proc_macro.rs +++ b/compiler/rustc_expand/src/proc_macro.rs @@ -133,11 +133,21 @@ impl MultiItemModifier for ProcMacroDerive { } } - // fail if there have been errors emitted, - // or if there's any unparsed tokens left. - if parser.token != token::Eof - || ecx.sess.parse_sess.span_diagnostic.err_count() > error_count_before - { + // check if the entire stream was parsed + if parser.token != token::Eof { + ecx.struct_span_err( + parser.token.span, + &format!( + "expected item, found {}", + rustc_parse::parser::token_descr(&parser.token) + ), + ) + .span_label(parser.token.span, "expected item") + .emit(); + } + + // fail if there have been errors emitted + if ecx.sess.parse_sess.span_diagnostic.err_count() > error_count_before { ecx.struct_span_err(span, "proc-macro derive produced unparseable tokens").emit(); } diff --git a/compiler/rustc_parse/src/parser/mod.rs b/compiler/rustc_parse/src/parser/mod.rs index 51d4e007b5984..d4877eb9c5eb1 100644 --- a/compiler/rustc_parse/src/parser/mod.rs +++ b/compiler/rustc_parse/src/parser/mod.rs @@ -394,7 +394,7 @@ fn token_descr_opt(token: &Token) -> Option<&'static str> { }) } -pub(super) fn token_descr(token: &Token) -> String { +pub fn token_descr(token: &Token) -> String { let token_str = pprust::token_to_string(token); match token_descr_opt(token) { Some(prefix) => format!("{} `{}`", prefix, token_str), diff --git a/src/test/ui/proc-macro/derive-bad.rs b/src/test/ui/proc-macro/derive-bad.rs index 8b097a40f704c..015fc2b7ea0aa 100644 --- a/src/test/ui/proc-macro/derive-bad.rs +++ b/src/test/ui/proc-macro/derive-bad.rs @@ -10,6 +10,7 @@ struct A; //~ ERROR the name `A` is defined multiple times #[derive(B)] //~^ ERROR proc-macro derive produced unparseable tokens +//~| ERROR expected item, found `{` struct B; fn main() {} diff --git a/src/test/ui/proc-macro/derive-bad.stderr b/src/test/ui/proc-macro/derive-bad.stderr index 9cdf4e3ab2456..7cd3175ac57ca 100644 --- a/src/test/ui/proc-macro/derive-bad.stderr +++ b/src/test/ui/proc-macro/derive-bad.stderr @@ -23,12 +23,20 @@ LL | struct A; | = note: `A` must be defined only once in the type namespace of this module +error: expected item, found `{` + --> $DIR/derive-bad.rs:11:10 + | +LL | #[derive(B)] + | ^ expected item + | + = note: this error originates in the derive macro `B` (in Nightly builds, run with -Z macro-backtrace for more info) + error: proc-macro derive produced unparseable tokens --> $DIR/derive-bad.rs:11:10 | LL | #[derive(B)] | ^ -error: aborting due to 4 previous errors +error: aborting due to 5 previous errors For more information about this error, try `rustc --explain E0428`. From 60241157408b47de43ebadd65f6b513687019535 Mon Sep 17 00:00:00 2001 From: Mara Bos Date: Wed, 28 Jul 2021 15:36:08 +0200 Subject: [PATCH 4/4] Split parse_items from parse_mod for proc_macro. --- compiler/rustc_expand/src/proc_macro.rs | 44 ++++++++----------------- compiler/rustc_parse/src/parser/item.rs | 6 +++- compiler/rustc_parse/src/parser/mod.rs | 2 +- 3 files changed, 20 insertions(+), 32 deletions(-) diff --git a/compiler/rustc_expand/src/proc_macro.rs b/compiler/rustc_expand/src/proc_macro.rs index d52484d9288ca..b72c2f91543b7 100644 --- a/compiler/rustc_expand/src/proc_macro.rs +++ b/compiler/rustc_expand/src/proc_macro.rs @@ -8,7 +8,6 @@ use rustc_ast::tokenstream::{CanSynthesizeMissingTokens, TokenStream, TokenTree} use rustc_data_structures::sync::Lrc; use rustc_errors::ErrorReported; use rustc_parse::nt_to_tokenstream; -use rustc_parse::parser::ForceCollect; use rustc_span::{Span, DUMMY_SP}; const EXEC_STRATEGY: pm::bridge::server::SameThread = pm::bridge::server::SameThread; @@ -114,37 +113,22 @@ impl MultiItemModifier for ProcMacroDerive { let error_count_before = ecx.sess.parse_sess.span_diagnostic.err_count(); let mut parser = rustc_parse::stream_to_parser(&ecx.sess.parse_sess, stream, Some("proc-macro derive")); - let mut items = vec![]; - loop { - match parser.parse_item(ForceCollect::No) { - Ok(None) => break, - Ok(Some(item)) => { - if is_stmt { - items.push(Annotatable::Stmt(P(ecx.stmt_item(span, item)))); - } else { - items.push(Annotatable::Item(item)); - } - } - Err(mut err) => { - err.emit(); - break; + let items = parser + .parse_items(&token::Eof) + .unwrap_or_else(|mut e| { + e.emit(); + Vec::new() + }) + .into_iter() + .map(|item| { + if is_stmt { + Annotatable::Stmt(P(ecx.stmt_item(span, item))) + } else { + Annotatable::Item(item) } - } - } - - // check if the entire stream was parsed - if parser.token != token::Eof { - ecx.struct_span_err( - parser.token.span, - &format!( - "expected item, found {}", - rustc_parse::parser::token_descr(&parser.token) - ), - ) - .span_label(parser.token.span, "expected item") - .emit(); - } + }) + .collect(); // fail if there have been errors emitted if ecx.sess.parse_sess.span_diagnostic.err_count() > error_count_before { diff --git a/compiler/rustc_parse/src/parser/item.rs b/compiler/rustc_parse/src/parser/item.rs index 2ce63d011f438..ff403a02c4ef6 100644 --- a/compiler/rustc_parse/src/parser/item.rs +++ b/compiler/rustc_parse/src/parser/item.rs @@ -54,7 +54,11 @@ impl<'a> Parser<'a> { ) -> PResult<'a, (Vec, Vec>, Span)> { let lo = self.token.span; let attrs = self.parse_inner_attributes()?; + let items = self.parse_items(term)?; + Ok((attrs, items, lo.to(self.prev_token.span))) + } + pub fn parse_items(&mut self, term: &TokenKind) -> PResult<'a, Vec>> { let mut items = vec![]; while let Some(item) = self.parse_item(ForceCollect::No)? { items.push(item); @@ -71,7 +75,7 @@ impl<'a> Parser<'a> { } } - Ok((attrs, items, lo.to(self.prev_token.span))) + Ok(items) } } diff --git a/compiler/rustc_parse/src/parser/mod.rs b/compiler/rustc_parse/src/parser/mod.rs index d4877eb9c5eb1..51d4e007b5984 100644 --- a/compiler/rustc_parse/src/parser/mod.rs +++ b/compiler/rustc_parse/src/parser/mod.rs @@ -394,7 +394,7 @@ fn token_descr_opt(token: &Token) -> Option<&'static str> { }) } -pub fn token_descr(token: &Token) -> String { +pub(super) fn token_descr(token: &Token) -> String { let token_str = pprust::token_to_string(token); match token_descr_opt(token) { Some(prefix) => format!("{} `{}`", prefix, token_str),