Skip to content

Commit c3da0d4

Browse files
nnethercotennethercote
committed
[passes ui tests] Add a skip field to Delimiter::Invisible.
Currently it's always set to `true`, which doesn't change existing behaviour.
1 parent 50cf76c commit c3da0d4

File tree

10 files changed

+35
-25
lines changed

10 files changed

+35
-25
lines changed

‎compiler/rustc_ast/src/ast.rs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1653,7 +1653,7 @@ impl MacDelimiter {
16531653
Delimiter::Parenthesis => Some(MacDelimiter::Parenthesis),
16541654
Delimiter::Bracket => Some(MacDelimiter::Bracket),
16551655
Delimiter::Brace => Some(MacDelimiter::Brace),
1656-
Delimiter::Invisible => None,
1656+
Delimiter::Invisible { .. } => None,
16571657
}
16581658
}
16591659
}

‎compiler/rustc_ast/src/attr/mod.rs

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -551,7 +551,7 @@ impl MetaItemKind {
551551
tokens: &mut impl Iterator<Item = TokenTree>,
552552
) -> Option<MetaItemKind> {
553553
match tokens.next() {
554-
Some(TokenTree::Delimited(_, Delimiter::Invisible, inner_tokens)) => {
554+
Some(TokenTree::Delimited(_, Delimiter::Invisible { .. }, inner_tokens)) => {
555555
MetaItemKind::name_value_from_tokens(&mut inner_tokens.trees())
556556
}
557557
Some(TokenTree::Token(token)) => {
@@ -621,7 +621,7 @@ impl NestedMetaItem {
621621
tokens.next();
622622
return Some(NestedMetaItem::Literal(lit));
623623
}
624-
Some(TokenTree::Delimited(_, Delimiter::Invisible, inner_tokens)) => {
624+
Some(TokenTree::Delimited(_, Delimiter::Invisible { .. }, inner_tokens)) => {
625625
let inner_tokens = inner_tokens.clone();
626626
tokens.next();
627627
return NestedMetaItem::from_tokens(&mut inner_tokens.into_trees().peekable());

‎compiler/rustc_ast/src/token.rs

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -56,7 +56,9 @@ pub enum Delimiter {
5656
/// `$var * 3` where `$var` is `1 + 2`.
5757
/// Invisible delimiters are not directly writable in normal Rust code except as comments.
5858
/// Therefore, they might not survive a roundtrip of a token stream through a string.
59-
Invisible,
59+
///
60+
/// njn: comment about `skip`
61+
Invisible { skip: bool },
6062
}
6163

6264
#[derive(Clone, Copy, PartialEq, Encodable, Decodable, Debug, HashStable_Generic)]

‎compiler/rustc_ast_pretty/src/pprust/state.rs

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -599,7 +599,7 @@ pub trait PrintState<'a>: std::ops::Deref<Target = pp::Printer> + std::ops::Dere
599599
self.end();
600600
self.bclose(span, empty);
601601
}
602-
Some(Delimiter::Invisible) => {
602+
Some(Delimiter::Invisible { .. }) => {
603603
self.word("/*«*/");
604604
let empty = tts.is_empty();
605605
if !empty {
@@ -786,8 +786,8 @@ pub trait PrintState<'a>: std::ops::Deref<Target = pp::Printer> + std::ops::Dere
786786
token::CloseDelim(Delimiter::Bracket) => "]".into(),
787787
token::OpenDelim(Delimiter::Brace) => "{".into(),
788788
token::CloseDelim(Delimiter::Brace) => "}".into(),
789-
token::OpenDelim(Delimiter::Invisible) => "/*«*/".into(),
790-
token::CloseDelim(Delimiter::Invisible) => "/*»*/".into(),
789+
token::OpenDelim(Delimiter::Invisible { .. }) => "/*«*/".into(),
790+
token::CloseDelim(Delimiter::Invisible { .. }) => "/*»*/".into(),
791791
token::Pound => "#".into(),
792792
token::Dollar => "$".into(),
793793
token::Question => "?".into(),

‎compiler/rustc_expand/src/mbe/quoted.rs

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -147,11 +147,14 @@ fn parse_tree(
147147
match tree {
148148
// `tree` is a `$` token. Look at the next token in `trees`
149149
tokenstream::TokenTree::Token(Token { kind: token::Dollar, span }) => {
150-
// FIXME: Handle `Invisible`-delimited groups in a more systematic way
151-
// during parsing.
152150
let mut next = outer_trees.next();
153151
let mut trees: Box<dyn Iterator<Item = tokenstream::TokenTree>>;
154-
if let Some(tokenstream::TokenTree::Delimited(_, Delimiter::Invisible, tts)) = next {
152+
if let Some(tokenstream::TokenTree::Delimited(
153+
_,
154+
Delimiter::Invisible { skip: true },
155+
tts,
156+
)) = next
157+
{
155158
trees = Box::new(tts.into_trees());
156159
next = trees.next();
157160
} else {

‎compiler/rustc_expand/src/proc_macro_server.rs

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -34,7 +34,7 @@ impl FromInternal<token::Delimiter> for Delimiter {
3434
token::Delimiter::Parenthesis => Delimiter::Parenthesis,
3535
token::Delimiter::Brace => Delimiter::Brace,
3636
token::Delimiter::Bracket => Delimiter::Bracket,
37-
token::Delimiter::Invisible => Delimiter::None,
37+
token::Delimiter::Invisible { .. } => Delimiter::None,
3838
}
3939
}
4040
}
@@ -45,7 +45,7 @@ impl ToInternal<token::Delimiter> for Delimiter {
4545
Delimiter::Parenthesis => token::Delimiter::Parenthesis,
4646
Delimiter::Brace => token::Delimiter::Brace,
4747
Delimiter::Bracket => token::Delimiter::Bracket,
48-
Delimiter::None => token::Delimiter::Invisible,
48+
Delimiter::None => token::Delimiter::Invisible { skip: true },
4949
}
5050
}
5151
}

‎compiler/rustc_parse/src/parser/attr_wrapper.rs

Lines changed: 6 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -414,11 +414,11 @@ fn make_token_stream(
414414
FlatToken::Token(Token { kind: TokenKind::CloseDelim(delim), span }) => {
415415
// HACK: If we encounter a mismatched `Invisible` delimiter at the top
416416
// level, just ignore it.
417-
if matches!(delim, Delimiter::Invisible)
417+
if matches!(delim, Delimiter::Invisible { .. })
418418
&& (stack.len() == 1
419419
|| !matches!(
420420
stack.last_mut().unwrap().open_delim_sp.unwrap().0,
421-
Delimiter::Invisible
421+
Delimiter::Invisible { .. }
422422
))
423423
{
424424
token_and_spacing = iter.next();
@@ -431,8 +431,8 @@ fn make_token_stream(
431431
// HACK: If our current frame has a mismatched opening `Invisible` delimiter,
432432
// merge our current frame with the one above it. That is, transform
433433
// `[ { < first second } third ]` into `[ { first second } third ]`
434-
if !matches!(delim, Delimiter::Invisible)
435-
&& matches!(frame_data.open_delim_sp.unwrap().0, Delimiter::Invisible)
434+
if !matches!(delim, Delimiter::Invisible { .. })
435+
&& matches!(frame_data.open_delim_sp.unwrap().0, Delimiter::Invisible { .. })
436436
{
437437
stack.last_mut().unwrap().inner.extend(frame_data.inner);
438438
// Process our closing delimiter again, this time at the previous
@@ -475,7 +475,8 @@ fn make_token_stream(
475475
// HACK: If we don't have a closing `Invisible` delimiter for our last
476476
// frame, merge the frame with the top-level frame. That is,
477477
// turn `< first second` into `first second`
478-
if stack.len() == 2 && stack[1].open_delim_sp.unwrap().0 == Delimiter::Invisible {
478+
if stack.len() == 2 && matches!(stack[1].open_delim_sp.unwrap().0, Delimiter::Invisible { .. })
479+
{
479480
let temp_buf = stack.pop().unwrap();
480481
stack.last_mut().unwrap().inner.extend(temp_buf.inner);
481482
}

‎compiler/rustc_parse/src/parser/diagnostics.rs

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2109,7 +2109,9 @@ impl<'a> Parser<'a> {
21092109
brace_depth -= 1;
21102110
continue;
21112111
}
2112-
} else if self.token == token::Eof || self.eat(&token::CloseDelim(Delimiter::Invisible))
2112+
// njn: ??
2113+
} else if self.token == token::Eof
2114+
|| self.eat(&token::CloseDelim(Delimiter::Invisible { skip: false }))
21132115
{
21142116
return;
21152117
} else {

‎compiler/rustc_parse/src/parser/mod.rs

Lines changed: 8 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -277,14 +277,14 @@ impl TokenCursor {
277277
// Set `open_delim` to true here because we deal with it immediately.
278278
let frame = TokenCursorFrame::new(Some((delim, sp)), tts.clone());
279279
self.stack.push(mem::replace(&mut self.frame, frame));
280-
if delim != Delimiter::Invisible {
280+
if delim != (Delimiter::Invisible { skip: true }) {
281281
return (Token::new(token::OpenDelim(delim), sp.open), Spacing::Alone);
282282
}
283283
// No open delimeter to return; continue on to the next iteration.
284284
}
285285
};
286286
} else if let Some(frame) = self.stack.pop() {
287-
if let Some((delim, span)) = self.frame.delim_sp && delim != Delimiter::Invisible {
287+
if let Some((delim, span)) = self.frame.delim_sp && delim != (Delimiter::Invisible { skip: true }) {
288288
self.frame = frame;
289289
return (Token::new(token::CloseDelim(delim), span.close), Spacing::Alone);
290290
}
@@ -1004,7 +1004,8 @@ impl<'a> Parser<'a> {
10041004
}
10051005
debug_assert!(!matches!(
10061006
next.0.kind,
1007-
token::OpenDelim(Delimiter::Invisible) | token::CloseDelim(Delimiter::Invisible)
1007+
token::OpenDelim(Delimiter::Invisible { skip: true })
1008+
| token::CloseDelim(Delimiter::Invisible { skip: true })
10081009
));
10091010
self.inlined_bump_with(next)
10101011
}
@@ -1017,10 +1018,10 @@ impl<'a> Parser<'a> {
10171018
}
10181019

10191020
let frame = &self.token_cursor.frame;
1020-
if let Some((delim, span)) = frame.delim_sp && delim != Delimiter::Invisible {
1021+
if let Some((delim, span)) = frame.delim_sp && (delim != Delimiter::Invisible { skip: true }) {
10211022
let all_normal = (0..dist).all(|i| {
10221023
let token = frame.tree_cursor.look_ahead(i);
1023-
!matches!(token, Some(TokenTree::Delimited(_, Delimiter::Invisible, _)))
1024+
!matches!(token, Some(TokenTree::Delimited(_, Delimiter::Invisible { skip: true }, _)))
10241025
});
10251026
if all_normal {
10261027
return match frame.tree_cursor.look_ahead(dist - 1) {
@@ -1042,7 +1043,8 @@ impl<'a> Parser<'a> {
10421043
token = cursor.next(/* desugar_doc_comments */ false).0;
10431044
if matches!(
10441045
token.kind,
1045-
token::OpenDelim(Delimiter::Invisible) | token::CloseDelim(Delimiter::Invisible)
1046+
token::OpenDelim(Delimiter::Invisible { skip: true })
1047+
| token::CloseDelim(Delimiter::Invisible { skip: true })
10461048
) {
10471049
continue;
10481050
}

‎compiler/rustc_session/src/utils.rs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -137,7 +137,7 @@ impl<'a> FlattenNonterminals<'a> {
137137
let tts = (self.nt_to_tokenstream)(&nt, self.parse_sess, self.synthesize_tokens);
138138
TokenTree::Delimited(
139139
DelimSpan::from_single(token.span),
140-
Delimiter::Invisible,
140+
Delimiter::Invisible { skip: true }, // njn: ?
141141
self.process_token_stream(tts),
142142
)
143143
.into()

0 commit comments

Comments
 (0)