Skip to content

Commit 6b6f97f

Browse files
nnethercotennethercote
committed
[has 23 ui test failures] Remove Nonterminal::NtExpr.
1 parent c3da0d4 commit 6b6f97f

File tree

20 files changed

+255
-195
lines changed

20 files changed

+255
-195
lines changed

‎compiler/rustc_ast/src/ast_like.rs

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -45,7 +45,7 @@ impl AstLike for crate::token::Nonterminal {
4545
match self {
4646
Nonterminal::NtItem(item) => item.attrs(),
4747
Nonterminal::NtStmt(stmt) => stmt.attrs(),
48-
Nonterminal::NtExpr(expr) | Nonterminal::NtLiteral(expr) => expr.attrs(),
48+
Nonterminal::NtLiteral(expr) => expr.attrs(),
4949
Nonterminal::NtPat(_)
5050
| Nonterminal::NtTy(_)
5151
| Nonterminal::NtMeta(_)
@@ -60,7 +60,7 @@ impl AstLike for crate::token::Nonterminal {
6060
match self {
6161
Nonterminal::NtItem(item) => item.visit_attrs(f),
6262
Nonterminal::NtStmt(stmt) => stmt.visit_attrs(f),
63-
Nonterminal::NtExpr(expr) | Nonterminal::NtLiteral(expr) => expr.visit_attrs(f),
63+
Nonterminal::NtLiteral(expr) => expr.visit_attrs(f),
6464
Nonterminal::NtPat(_)
6565
| Nonterminal::NtTy(_)
6666
| Nonterminal::NtMeta(_)
@@ -75,7 +75,7 @@ impl AstLike for crate::token::Nonterminal {
7575
match self {
7676
Nonterminal::NtItem(item) => item.tokens_mut(),
7777
Nonterminal::NtStmt(stmt) => stmt.tokens_mut(),
78-
Nonterminal::NtExpr(expr) | Nonterminal::NtLiteral(expr) => expr.tokens_mut(),
78+
Nonterminal::NtLiteral(expr) => expr.tokens_mut(),
7979
Nonterminal::NtPat(pat) => pat.tokens_mut(),
8080
Nonterminal::NtTy(ty) => ty.tokens_mut(),
8181
Nonterminal::NtMeta(attr_item) => attr_item.tokens_mut(),

‎compiler/rustc_ast/src/mut_visit.rs

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -776,7 +776,6 @@ pub fn visit_nonterminal<T: MutVisitor>(nt: &mut token::Nonterminal, vis: &mut T
776776
})
777777
}),
778778
token::NtPat(pat) => vis.visit_pat(pat),
779-
token::NtExpr(expr) => vis.visit_expr(expr),
780779
token::NtTy(ty) => vis.visit_ty(ty),
781780
token::NtIdent(ident, _is_raw) => vis.visit_ident(ident),
782781
token::NtLifetime(ident) => vis.visit_ident(ident),

‎compiler/rustc_ast/src/token.rs

Lines changed: 8 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -379,7 +379,7 @@ impl Token {
379379
match self.uninterpolate().kind {
380380
Ident(name, is_raw) =>
381381
ident_can_begin_expr(name, self.span, is_raw), // value name or keyword
382-
OpenDelim(..) | // tuple, array or block
382+
OpenDelim(..) | // tuple, array, block, or macro output
383383
Literal(..) | // literal
384384
Not | // operator not
385385
BinOp(Minus) | // unary minus
@@ -394,7 +394,6 @@ impl Token {
394394
Lifetime(..) | // labeled loop
395395
Pound => true, // expression attributes
396396
Interpolated(ref nt) => matches!(**nt, NtLiteral(..) |
397-
NtExpr(..) |
398397
NtBlock(..) |
399398
NtPath(..)),
400399
_ => false,
@@ -424,8 +423,8 @@ impl Token {
424423
/// Returns `true` if the token can appear at the start of a const param.
425424
pub fn can_begin_const_arg(&self) -> bool {
426425
match self.kind {
427-
OpenDelim(Delimiter::Brace) => true,
428-
Interpolated(ref nt) => matches!(**nt, NtExpr(..) | NtBlock(..) | NtLiteral(..)),
426+
OpenDelim(Delimiter::Brace | Delimiter::Invisible { .. }) => true,
427+
Interpolated(ref nt) => matches!(**nt, NtBlock(..) | NtLiteral(..)),
429428
_ => self.can_begin_literal_maybe_minus(),
430429
}
431430
}
@@ -454,17 +453,8 @@ impl Token {
454453
match self.uninterpolate().kind {
455454
Literal(..) | BinOp(Minus) => true,
456455
Ident(name, false) if name.is_bool_lit() => true,
457-
Interpolated(ref nt) => match &**nt {
458-
NtLiteral(_) => true,
459-
NtExpr(e) => match &e.kind {
460-
ast::ExprKind::Lit(_) => true,
461-
ast::ExprKind::Unary(ast::UnOp::Neg, e) => {
462-
matches!(&e.kind, ast::ExprKind::Lit(_))
463-
}
464-
_ => false,
465-
},
466-
_ => false,
467-
},
456+
OpenDelim(Delimiter::Invisible { .. }) => true,
457+
Interpolated(ref nt) => matches!(**nt, NtLiteral(_)),
468458
_ => false,
469459
}
470460
}
@@ -541,9 +531,10 @@ impl Token {
541531
/// Would `maybe_whole_expr` in `parser.rs` return `Ok(..)`?
542532
/// That is, is this a pre-parsed expression dropped into the token stream
543533
/// (which happens while parsing the result of macro expansion)?
534+
// njn: need to do anything with this?
544535
pub fn is_whole_expr(&self) -> bool {
545536
if let Interpolated(ref nt) = self.kind
546-
&& let NtExpr(_) | NtLiteral(_) | NtPath(_) | NtBlock(_) = **nt
537+
&& let NtLiteral(_) | NtPath(_) | NtBlock(_) = **nt
547538
{
548539
return true;
549540
}
@@ -699,7 +690,6 @@ pub enum Nonterminal {
699690
NtBlock(P<ast::Block>),
700691
NtStmt(P<ast::Stmt>),
701692
NtPat(P<ast::Pat>),
702-
NtExpr(P<ast::Expr>),
703693
NtTy(P<ast::Ty>),
704694
NtIdent(Ident, /* is_raw */ bool),
705695
NtLifetime(Ident),
@@ -799,7 +789,7 @@ impl Nonterminal {
799789
NtBlock(block) => block.span,
800790
NtStmt(stmt) => stmt.span,
801791
NtPat(pat) => pat.span,
802-
NtExpr(expr) | NtLiteral(expr) => expr.span,
792+
NtLiteral(expr) => expr.span,
803793
NtTy(ty) => ty.span,
804794
NtIdent(ident, _) | NtLifetime(ident) => ident.span,
805795
NtMeta(attr_item) => attr_item.span(),
@@ -832,7 +822,6 @@ impl fmt::Debug for Nonterminal {
832822
NtBlock(..) => f.pad("NtBlock(..)"),
833823
NtStmt(..) => f.pad("NtStmt(..)"),
834824
NtPat(..) => f.pad("NtPat(..)"),
835-
NtExpr(..) => f.pad("NtExpr(..)"),
836825
NtTy(..) => f.pad("NtTy(..)"),
837826
NtIdent(..) => f.pad("NtIdent(..)"),
838827
NtLiteral(..) => f.pad("NtLiteral(..)"),

‎compiler/rustc_ast/src/util/literal.rs

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -215,14 +215,16 @@ impl Lit {
215215
/// Converts arbitrary token into an AST literal.
216216
///
217217
/// Keep this in sync with `Token::can_begin_literal_or_bool` excluding unary negation.
218+
// njn: need to do something here? It's hard to keep in sync with
219+
// can_begin_literal_or_bool when a literal can span 3 tokens: `«`, `lit`, `»`
218220
pub fn from_token(token: &Token) -> Result<Lit, LitError> {
219221
let lit = match token.uninterpolate().kind {
220222
token::Ident(name, false) if name.is_bool_lit() => {
221223
token::Lit::new(token::Bool, name, None)
222224
}
223225
token::Literal(lit) => lit,
224226
token::Interpolated(ref nt) => {
225-
if let token::NtExpr(expr) | token::NtLiteral(expr) = &**nt
227+
if let token::NtLiteral(expr) = &**nt
226228
&& let ast::ExprKind::Lit(lit) = &expr.kind
227229
{
228230
return Ok(lit.clone());

‎compiler/rustc_ast_pretty/src/pprust/state.rs

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -727,7 +727,6 @@ pub trait PrintState<'a>: std::ops::Deref<Target = pp::Printer> + std::ops::Dere
727727

728728
fn nonterminal_to_string(&self, nt: &Nonterminal) -> String {
729729
match *nt {
730-
token::NtExpr(ref e) => self.expr_to_string(e),
731730
token::NtMeta(ref e) => self.attr_item_to_string(e),
732731
token::NtTy(ref e) => self.ty_to_string(e),
733732
token::NtPath(ref e) => self.path_to_string(e),

‎compiler/rustc_builtin_macros/src/cfg_eval.rs

Lines changed: 1 addition & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -173,13 +173,7 @@ impl CfgEval<'_, '_> {
173173
}
174174
_ => unreachable!(),
175175
};
176-
let nt = annotatable.into_nonterminal();
177-
178-
let mut orig_tokens = rustc_parse::nt_to_tokenstream(
179-
&nt,
180-
&self.cfg.sess.parse_sess,
181-
CanSynthesizeMissingTokens::No,
182-
);
176+
let mut orig_tokens = annotatable.into_tokens(&self.cfg.sess.parse_sess);
183177

184178
// 'Flatten' all nonterminals (i.e. `TokenKind::Interpolated`)
185179
// to `None`-delimited groups containing the corresponding tokens. This

‎compiler/rustc_expand/src/base.rs

Lines changed: 8 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,7 @@ use rustc_data_structures::sync::{self, Lrc};
1313
use rustc_errors::{Applicability, DiagnosticBuilder, ErrorGuaranteed, MultiSpan, PResult};
1414
use rustc_lint_defs::builtin::PROC_MACRO_BACK_COMPAT;
1515
use rustc_lint_defs::BuiltinLintDiagnostics;
16-
use rustc_parse::{self, nt_to_tokenstream, parser, MACRO_ARGUMENTS};
16+
use rustc_parse::{self, expr_to_tokenstream, nt_to_tokenstream, parser, MACRO_ARGUMENTS};
1717
use rustc_session::{parse::ParseSess, Limit, Session};
1818
use rustc_span::def_id::{CrateNum, DefId, LocalDefId};
1919
use rustc_span::edition::Edition;
@@ -119,8 +119,8 @@ impl Annotatable {
119119
token::NtItem(P(item.and_then(ast::ForeignItem::into_item)))
120120
}
121121
Annotatable::Stmt(stmt) => token::NtStmt(stmt),
122-
Annotatable::Expr(expr) => token::NtExpr(expr),
123-
Annotatable::Arm(..)
122+
Annotatable::Expr(..)
123+
| Annotatable::Arm(..)
124124
| Annotatable::ExprField(..)
125125
| Annotatable::PatField(..)
126126
| Annotatable::GenericParam(..)
@@ -131,8 +131,11 @@ impl Annotatable {
131131
}
132132
}
133133

134-
crate fn into_tokens(self, sess: &ParseSess) -> TokenStream {
135-
nt_to_tokenstream(&self.into_nonterminal(), sess, CanSynthesizeMissingTokens::No)
134+
pub fn into_tokens(self, sess: &ParseSess) -> TokenStream {
135+
match self {
136+
Annotatable::Expr(expr) => expr_to_tokenstream(&expr, sess, CanSynthesizeMissingTokens::No),
137+
_ => nt_to_tokenstream(&self.into_nonterminal(), sess, CanSynthesizeMissingTokens::No),
138+
}
136139
}
137140

138141
pub fn expect_item(self) -> P<ast::Item> {

‎compiler/rustc_expand/src/mbe/macro_parser.rs

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -75,6 +75,7 @@ crate use ParseResult::*;
7575

7676
use crate::mbe::{KleeneOp, TokenTree};
7777

78+
use rustc_ast as ast;
7879
use rustc_ast::token::{self, DocComment, Nonterminal, NonterminalKind, Token};
7980
use rustc_parse::parser::{NtOrTt, Parser};
8081
use rustc_span::symbol::MacroRulesNormalizedIdent;
@@ -344,6 +345,9 @@ crate enum NamedMatch {
344345
// A metavar match of type `tt`.
345346
MatchedTokenTree(rustc_ast::tokenstream::TokenTree),
346347

348+
// njn: comment
349+
MatchedExpr(ast::ptr::P<ast::Expr>), // njn: quals
350+
347351
// A metavar match of any type other than `tt`.
348352
MatchedNonterminal(Lrc<Nonterminal>),
349353
}
@@ -624,6 +628,7 @@ impl TtParser {
624628
let m = match nt {
625629
NtOrTt::Nt(nt) => MatchedNonterminal(Lrc::new(nt)),
626630
NtOrTt::Tt(tt) => MatchedTokenTree(tt),
631+
NtOrTt::Expr(e) => MatchedExpr(e),
627632
};
628633
mp.push_match(next_metavar, seq_depth, m);
629634
mp.idx += 1;

‎compiler/rustc_expand/src/mbe/transcribe.rs

Lines changed: 27 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,12 +1,14 @@
11
use crate::base::ExtCtxt;
2-
use crate::mbe::macro_parser::{MatchedNonterminal, MatchedSeq, MatchedTokenTree, NamedMatch};
2+
use crate::mbe::macro_parser::{NamedMatch, NamedMatch::*};
33
use crate::mbe::{self, MetaVarExpr};
44
use rustc_ast::mut_visit::{self, MutVisitor};
55
use rustc_ast::token::{self, Delimiter, Token, TokenKind};
6-
use rustc_ast::tokenstream::{DelimSpan, TokenStream, TokenTree, TreeAndSpacing};
6+
use rustc_ast::tokenstream::CanSynthesizeMissingTokens;
7+
use rustc_ast::tokenstream::{DelimSpan, Spacing, TokenStream, TokenTree, TreeAndSpacing};
78
use rustc_data_structures::fx::FxHashMap;
89
use rustc_errors::{pluralize, PResult};
910
use rustc_errors::{DiagnosticBuilder, ErrorGuaranteed};
11+
use rustc_parse::expr_to_tokenstream;
1012
use rustc_span::hygiene::{LocalExpnId, Transparency};
1113
use rustc_span::symbol::{sym, Ident, MacroRulesNormalizedIdent};
1214
use rustc_span::Span;
@@ -230,13 +232,31 @@ pub(super) fn transcribe<'a>(
230232
result.push(token.into());
231233
}
232234
MatchedNonterminal(ref nt) => {
235+
// njn: update comment
233236
// Other variables are emitted into the output stream as groups with
234237
// `Delimiter::Invisible` to maintain parsing priorities.
235238
// `Interpolated` is currently used for such groups in rustc parser.
236239
marker.visit_span(&mut sp);
237-
let token = TokenTree::token(token::Interpolated(nt.clone()), sp);
240+
let token =
241+
TokenTree::token(token::Interpolated(nt.clone()), sp);
238242
result.push(token.into());
239243
}
244+
MatchedExpr(ref e) => {
245+
// njn: update comment
246+
let tts = expr_to_tokenstream(
247+
e,
248+
&cx.sess.parse_sess,
249+
CanSynthesizeMissingTokens::No,
250+
);
251+
marker.visit_span(&mut sp);
252+
let tt = TokenTree::Delimited(
253+
DelimSpan::from_single(sp),
254+
// njn: `skip: false`!
255+
Delimiter::Invisible { skip: false },
256+
tts,
257+
);
258+
result.push((tt, Spacing::Alone));
259+
}
240260
MatchedSeq(..) => {
241261
// We were unable to descend far enough. This is an error.
242262
return Err(cx.struct_span_err(
@@ -306,7 +326,7 @@ fn lookup_cur_matched<'a>(
306326
let mut matched = matched;
307327
for &(idx, _) in repeats {
308328
match matched {
309-
MatchedTokenTree(_) | MatchedNonterminal(_) => break,
329+
MatchedTokenTree(_) | MatchedNonterminal(_) | MatchedExpr(_) => break,
310330
MatchedSeq(ref ads) => matched = ads.get(idx).unwrap(),
311331
}
312332
}
@@ -396,7 +416,7 @@ fn lockstep_iter_size(
396416
let name = MacroRulesNormalizedIdent::new(name);
397417
match lookup_cur_matched(name, interpolations, repeats) {
398418
Some(matched) => match matched {
399-
MatchedTokenTree(_) | MatchedNonterminal(_) => LockstepIterSize::Unconstrained,
419+
MatchedTokenTree(_) | MatchedNonterminal(_) | MatchedExpr(_) => LockstepIterSize::Unconstrained,
400420
MatchedSeq(ref ads) => LockstepIterSize::Constraint(ads.len(), name),
401421
},
402422
_ => LockstepIterSize::Unconstrained,
@@ -443,7 +463,8 @@ fn count_repetitions<'a>(
443463
sp: &DelimSpan,
444464
) -> PResult<'a, usize> {
445465
match matched {
446-
MatchedTokenTree(_) | MatchedNonterminal(_) => {
466+
// njn: flip order of these arms, and in similar matches above
467+
MatchedTokenTree(_) | MatchedNonterminal(_) | MatchedExpr(_) => {
447468
if declared_lhs_depth == 0 {
448469
return Err(cx.struct_span_err(
449470
sp.entire(),

‎compiler/rustc_parse/src/lib.rs

Lines changed: 29 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -290,15 +290,15 @@ pub fn nt_to_tokenstream(
290290
Nonterminal::NtMeta(ref attr) => convert_tokens(attr.tokens.as_ref()),
291291
Nonterminal::NtPath(ref path) => convert_tokens(path.tokens.as_ref()),
292292
Nonterminal::NtVis(ref vis) => convert_tokens(vis.tokens.as_ref()),
293-
Nonterminal::NtExpr(ref expr) | Nonterminal::NtLiteral(ref expr) => {
293+
Nonterminal::NtLiteral(ref expr) => {
294294
prepend_attrs(&expr.attrs, expr.tokens.as_ref())
295295
}
296296
};
297297

298298
if let Some(tokens) = tokens {
299-
return tokens;
299+
tokens
300300
} else if matches!(synthesize_tokens, CanSynthesizeMissingTokens::Yes) {
301-
return fake_token_stream(sess, nt);
301+
fake_token_stream(sess, nt)
302302
} else {
303303
panic!(
304304
"Missing tokens for nt {:?} at {:?}: {:?}",
@@ -309,6 +309,26 @@ pub fn nt_to_tokenstream(
309309
}
310310
}
311311

312+
// njn: comments
313+
pub fn expr_to_tokenstream(
314+
expr: &ast::Expr,
315+
sess: &ParseSess,
316+
synthesize_tokens: CanSynthesizeMissingTokens,
317+
) -> TokenStream {
318+
if let Some(tokens) = prepend_attrs(&expr.attrs, expr.tokens.as_ref()) {
319+
tokens
320+
} else if matches!(synthesize_tokens, CanSynthesizeMissingTokens::Yes) {
321+
fake_token_stream_expr(sess, expr)
322+
} else {
323+
panic!(
324+
"Missing tokens for expr {:?} at {:?}: {:?}",
325+
expr,
326+
expr.span,
327+
pprust::expr_to_string(expr)
328+
);
329+
}
330+
}
331+
312332
fn prepend_attrs(attrs: &[Attribute], tokens: Option<&LazyTokenStream>) -> Option<TokenStream> {
313333
let tokens = tokens?;
314334
if attrs.is_empty() {
@@ -328,6 +348,12 @@ pub fn fake_token_stream(sess: &ParseSess, nt: &Nonterminal) -> TokenStream {
328348
parse_stream_from_source_str(filename, source, sess, Some(nt.span()))
329349
}
330350

351+
pub fn fake_token_stream_expr(sess: &ParseSess, expr: &ast::Expr) -> TokenStream {
352+
let source = pprust::expr_to_string(expr);
353+
let filename = FileName::macro_expansion_source_code(&source);
354+
parse_stream_from_source_str(filename, source, sess, Some(expr.span))
355+
}
356+
331357
pub fn fake_token_stream_for_crate(sess: &ParseSess, krate: &ast::Crate) -> TokenStream {
332358
let source = pprust::crate_to_string_for_macros(krate);
333359
let filename = FileName::macro_expansion_source_code(&source);

0 commit comments

Comments
 (0)