diff --git a/compiler/rustc_ast/src/attr/mod.rs b/compiler/rustc_ast/src/attr/mod.rs index 84654a9f73781..b7091ea1864d7 100644 --- a/compiler/rustc_ast/src/attr/mod.rs +++ b/compiler/rustc_ast/src/attr/mod.rs @@ -552,7 +552,7 @@ impl MetaItemKind { ) -> Option { match tokens.next() { Some(TokenTree::Delimited(_, Delimiter::Invisible, inner_tokens)) => { - MetaItemKind::name_value_from_tokens(&mut inner_tokens.trees()) + MetaItemKind::name_value_from_tokens(&mut inner_tokens.into_trees()) } Some(TokenTree::Token(token)) => { Lit::from_token(&token).ok().map(MetaItemKind::NameValue) diff --git a/compiler/rustc_ast/src/tokenstream.rs b/compiler/rustc_ast/src/tokenstream.rs index a8f29f334070e..23a039ec86812 100644 --- a/compiler/rustc_ast/src/tokenstream.rs +++ b/compiler/rustc_ast/src/tokenstream.rs @@ -442,8 +442,8 @@ impl TokenStream { } } - pub fn trees(&self) -> Cursor { - self.clone().into_trees() + pub fn trees(&self) -> CursorRef<'_> { + CursorRef::new(self) } pub fn into_trees(self) -> Cursor { @@ -538,12 +538,21 @@ pub struct CursorRef<'t> { } impl<'t> CursorRef<'t> { + fn new(stream: &'t TokenStream) -> Self { + CursorRef { stream, index: 0 } + } + + #[inline] fn next_with_spacing(&mut self) -> Option<&'t TreeAndSpacing> { self.stream.0.get(self.index).map(|tree| { self.index += 1; tree }) } + + pub fn look_ahead(&self, n: usize) -> Option<&TokenTree> { + self.stream.0[self.index..].get(n).map(|(tree, _)| tree) + } } impl<'t> Iterator for CursorRef<'t> { diff --git a/compiler/rustc_ast_pretty/src/pprust/state.rs b/compiler/rustc_ast_pretty/src/pprust/state.rs index f520b54112464..ed0177a5b4da1 100644 --- a/compiler/rustc_ast_pretty/src/pprust/state.rs +++ b/compiler/rustc_ast_pretty/src/pprust/state.rs @@ -550,9 +550,9 @@ pub trait PrintState<'a>: std::ops::Deref + std::ops::Dere fn print_tts(&mut self, tts: &TokenStream, convert_dollar_crate: bool) { let mut iter = tts.trees().peekable(); while let Some(tt) = iter.next() { - self.print_tt(&tt, convert_dollar_crate); + self.print_tt(tt, convert_dollar_crate); if let Some(next) = iter.peek() { - if tt_prepend_space(next, &tt) { + if tt_prepend_space(next, tt) { self.space(); } } diff --git a/compiler/rustc_expand/src/config.rs b/compiler/rustc_expand/src/config.rs index 0b8cb07a64afc..41683db80a314 100644 --- a/compiler/rustc_expand/src/config.rs +++ b/compiler/rustc_expand/src/config.rs @@ -400,7 +400,7 @@ impl<'a> StripUnconfigured<'a> { // Use the `#` in `#[cfg_attr(pred, attr)]` as the `#` token // for `attr` when we expand it to `#[attr]` - let mut orig_trees = orig_tokens.trees(); + let mut orig_trees = orig_tokens.into_trees(); let TokenTree::Token(pound_token @ Token { kind: TokenKind::Pound, .. }) = orig_trees.next().unwrap() else { panic!("Bad tokens for attribute {:?}", attr); }; diff --git a/compiler/rustc_expand/src/mbe/metavar_expr.rs b/compiler/rustc_expand/src/mbe/metavar_expr.rs index cdc5e204236fb..ceeb9baff4b56 100644 --- a/compiler/rustc_expand/src/mbe/metavar_expr.rs +++ b/compiler/rustc_expand/src/mbe/metavar_expr.rs @@ -1,5 +1,5 @@ use rustc_ast::token::{self, Delimiter}; -use rustc_ast::tokenstream::{Cursor, TokenStream, TokenTree}; +use rustc_ast::tokenstream::{CursorRef, TokenStream, TokenTree}; use rustc_ast::{LitIntType, LitKind}; use rustc_ast_pretty::pprust; use rustc_errors::{Applicability, PResult}; @@ -71,12 +71,14 @@ impl MetaVarExpr { } // Checks if there are any remaining tokens. For example, `${ignore(ident ... a b c ...)}` -fn check_trailing_token<'sess>(iter: &mut Cursor, sess: &'sess ParseSess) -> PResult<'sess, ()> { +fn check_trailing_token<'sess>( + iter: &mut CursorRef<'_>, + sess: &'sess ParseSess, +) -> PResult<'sess, ()> { if let Some(tt) = iter.next() { - let mut diag = sess.span_diagnostic.struct_span_err( - tt.span(), - &format!("unexpected token: {}", pprust::tt_to_string(&tt)), - ); + let mut diag = sess + .span_diagnostic + .struct_span_err(tt.span(), &format!("unexpected token: {}", pprust::tt_to_string(tt))); diag.span_note(tt.span(), "meta-variable expression must not have trailing tokens"); Err(diag) } else { @@ -86,7 +88,7 @@ fn check_trailing_token<'sess>(iter: &mut Cursor, sess: &'sess ParseSess) -> PRe /// Parse a meta-variable `count` expression: `count(ident[, depth])` fn parse_count<'sess>( - iter: &mut Cursor, + iter: &mut CursorRef<'_>, sess: &'sess ParseSess, span: Span, ) -> PResult<'sess, MetaVarExpr> { @@ -97,7 +99,7 @@ fn parse_count<'sess>( /// Parses the depth used by index(depth) and length(depth). fn parse_depth<'sess>( - iter: &mut Cursor, + iter: &mut CursorRef<'_>, sess: &'sess ParseSess, span: Span, ) -> PResult<'sess, usize> { @@ -110,7 +112,7 @@ fn parse_depth<'sess>( "meta-variable expression depth must be a literal" )); }; - if let Ok(lit_kind) = LitKind::from_lit_token(lit) + if let Ok(lit_kind) = LitKind::from_lit_token(*lit) && let LitKind::Int(n_u128, LitIntType::Unsuffixed) = lit_kind && let Ok(n_usize) = usize::try_from(n_u128) { @@ -124,7 +126,7 @@ fn parse_depth<'sess>( /// Parses an generic ident fn parse_ident<'sess>( - iter: &mut Cursor, + iter: &mut CursorRef<'_>, sess: &'sess ParseSess, span: Span, ) -> PResult<'sess, Ident> { @@ -132,7 +134,7 @@ fn parse_ident<'sess>( if let Some((elem, false)) = token.ident() { return Ok(elem); } - let token_str = pprust::token_to_string(&token); + let token_str = pprust::token_to_string(token); let mut err = sess.span_diagnostic.struct_span_err( span, &format!("expected identifier, found `{}`", &token_str) @@ -150,7 +152,7 @@ fn parse_ident<'sess>( /// Tries to move the iterator forward returning `true` if there is a comma. If not, then the /// iterator is not modified and the result is `false`. -fn try_eat_comma(iter: &mut Cursor) -> bool { +fn try_eat_comma(iter: &mut CursorRef<'_>) -> bool { if let Some(TokenTree::Token(token::Token { kind: token::Comma, .. })) = iter.look_ahead(0) { let _ = iter.next(); return true; diff --git a/compiler/rustc_expand/src/mbe/quoted.rs b/compiler/rustc_expand/src/mbe/quoted.rs index d52de24c393b7..707cb73f097f8 100644 --- a/compiler/rustc_expand/src/mbe/quoted.rs +++ b/compiler/rustc_expand/src/mbe/quoted.rs @@ -48,7 +48,7 @@ pub(super) fn parse( // For each token tree in `input`, parse the token into a `self::TokenTree`, consuming // additional trees if need be. - let mut trees = input.trees(); + let mut trees = input.into_trees(); while let Some(tree) = trees.next() { // Given the parsed tree, if there is a metavar and we are expecting matchers, actually // parse out the matcher (i.e., in `$id:ident` this would parse the `:` and `ident`). diff --git a/compiler/rustc_expand/src/parse/tests.rs b/compiler/rustc_expand/src/parse/tests.rs index 5d447d911e7f4..8da7879275895 100644 --- a/compiler/rustc_expand/src/parse/tests.rs +++ b/compiler/rustc_expand/src/parse/tests.rs @@ -61,7 +61,7 @@ fn bad_path_expr_1() { fn string_to_tts_macro() { create_default_session_globals_then(|| { let tts: Vec<_> = - string_to_stream("macro_rules! zip (($a)=>($a))".to_string()).trees().collect(); + string_to_stream("macro_rules! zip (($a)=>($a))".to_string()).into_trees().collect(); let tts: &[TokenTree] = &tts[..]; match tts { @@ -293,7 +293,7 @@ fn ttdelim_span() { .unwrap(); let tts: Vec<_> = match expr.kind { - ast::ExprKind::MacCall(ref mac) => mac.args.inner_tokens().trees().collect(), + ast::ExprKind::MacCall(ref mac) => mac.args.inner_tokens().into_trees().collect(), _ => panic!("not a macro"), }; diff --git a/compiler/rustc_expand/src/proc_macro_server.rs b/compiler/rustc_expand/src/proc_macro_server.rs index c0c786e4712e5..5b4c65a306b95 100644 --- a/compiler/rustc_expand/src/proc_macro_server.rs +++ b/compiler/rustc_expand/src/proc_macro_server.rs @@ -484,7 +484,7 @@ impl server::TokenStream for Rustc<'_, '_> { tree.to_internal() } fn into_iter(&mut self, stream: Self::TokenStream) -> Self::TokenStreamIter { - TokenStreamIter { cursor: stream.trees(), stack: vec![] } + TokenStreamIter { cursor: stream.into_trees(), stack: vec![] } } } diff --git a/compiler/rustc_expand/src/tokenstream/tests.rs b/compiler/rustc_expand/src/tokenstream/tests.rs index 31052bfb54ce8..270532f8edeec 100644 --- a/compiler/rustc_expand/src/tokenstream/tests.rs +++ b/compiler/rustc_expand/src/tokenstream/tests.rs @@ -35,7 +35,7 @@ fn test_concat() { fn test_to_from_bijection() { create_default_session_globals_then(|| { let test_start = string_to_ts("foo::bar(baz)"); - let test_end = test_start.trees().collect(); + let test_end = test_start.trees().cloned().collect(); assert_eq!(test_start, test_end) }) } diff --git a/compiler/rustc_session/src/utils.rs b/compiler/rustc_session/src/utils.rs index db755ccd1d515..9fda5373fcc1c 100644 --- a/compiler/rustc_session/src/utils.rs +++ b/compiler/rustc_session/src/utils.rs @@ -108,7 +108,7 @@ impl<'a> FlattenNonterminals<'a> { fn can_skip(stream: &TokenStream) -> bool { stream.trees().all(|tree| match tree { TokenTree::Token(token) => !matches!(token.kind, token::Interpolated(_)), - TokenTree::Delimited(_, _, inner) => can_skip(&inner), + TokenTree::Delimited(_, _, inner) => can_skip(inner), }) } diff --git a/src/tools/rustfmt/src/macros.rs b/src/tools/rustfmt/src/macros.rs index 26c429eb94ff3..f4b2bcf281577 100644 --- a/src/tools/rustfmt/src/macros.rs +++ b/src/tools/rustfmt/src/macros.rs @@ -226,7 +226,7 @@ fn rewrite_macro_inner( } // Format well-known macros which cannot be parsed as a valid AST. if macro_name == "lazy_static!" && !has_comment { - if let success @ Some(..) = format_lazy_static(context, shape, ts.trees().collect()) { + if let success @ Some(..) = format_lazy_static(context, shape, ts.clone()) { return success; } } @@ -855,7 +855,7 @@ impl MacroArgParser { /// Returns a collection of parsed macro def's arguments. fn parse(mut self, tokens: TokenStream) -> Option> { - let mut iter = tokens.trees(); + let mut iter = tokens.into_trees(); while let Some(tok) = iter.next() { match tok {