Remove Peekable<TokenStreamIter> uses.

Currently there are two ways to peek at a `TokenStreamIter`.
- Wrap it in a `Peekable` and use that traits `peek` method.
- Use `TokenStreamIter`'s inherent `peek` method.

Some code uses one, some use the other. This commit converts all places
to the inherent method. This eliminates mixing of `TokenStreamIter` and
`Peekable<TokenStreamIter>` and some use of `impl Iterator` and `dyn
Iterator`.

Author: nnethercote

compiler/rustc_ast/src/attr/mod.rs

@@ -1,6 +1,5 @@
 //! Functions dealing with attributes and meta items.
 
-use std::iter;
 use std::sync::atomic::{AtomicU32, Ordering};
 
 use rustc_index::bit_set::GrowableBitSet;
@@ -16,7 +15,9 @@ use crate::ast::{
 };
 use crate::ptr::P;
 use crate::token::{self, CommentKind, Delimiter, Token};
-use crate::tokenstream::{DelimSpan, LazyAttrTokenStream, Spacing, TokenStream, TokenTree};
+use crate::tokenstream::{
+    DelimSpan, LazyAttrTokenStream, Spacing, TokenStream, TokenStreamIter, TokenTree,
+};
 use crate::util::comments;
 use crate::util::literal::escape_string_symbol;
 
@@ -368,10 +369,7 @@ impl MetaItem {
         }
     }
 
-    fn from_tokens<'a, I>(iter: &mut iter::Peekable<I>) -> Option<MetaItem>
-    where
-        I: Iterator<Item = &'a TokenTree>,
-    {
+    fn from_tokens(iter: &mut TokenStreamIter<'_>) -> Option<MetaItem> {
         // FIXME: Share code with `parse_path`.
         let tt = iter.next().map(|tt| TokenTree::uninterpolate(tt));
         let path = match tt.as_deref() {
@@ -440,7 +438,7 @@ impl MetaItem {
 
 impl MetaItemKind {
     fn list_from_tokens(tokens: TokenStream) -> Option<ThinVec<MetaItemInner>> {
-        let mut iter = tokens.iter().peekable();
+        let mut iter = tokens.iter();
         let mut result = ThinVec::new();
         while iter.peek().is_some() {
             let item = MetaItemInner::from_tokens(&mut iter)?;
@@ -453,9 +451,7 @@ impl MetaItemKind {
         Some(result)
     }
 
-    fn name_value_from_tokens<'a>(
-        iter: &mut impl Iterator<Item = &'a TokenTree>,
-    ) -> Option<MetaItemKind> {
+    fn name_value_from_tokens(iter: &mut TokenStreamIter<'_>) -> Option<MetaItemKind> {
         match iter.next() {
             Some(TokenTree::Delimited(.., Delimiter::Invisible(_), inner_tokens)) => {
                 MetaItemKind::name_value_from_tokens(&mut inner_tokens.iter())
@@ -467,9 +463,7 @@ impl MetaItemKind {
         }
     }
 
-    fn from_tokens<'a>(
-        iter: &mut iter::Peekable<impl Iterator<Item = &'a TokenTree>>,
-    ) -> Option<MetaItemKind> {
+    fn from_tokens(iter: &mut TokenStreamIter<'_>) -> Option<MetaItemKind> {
         match iter.peek() {
             Some(TokenTree::Delimited(.., Delimiter::Parenthesis, inner_tokens)) => {
                 let inner_tokens = inner_tokens.clone();
@@ -598,18 +592,15 @@ impl MetaItemInner {
         self.meta_item().is_some()
     }
 
-    fn from_tokens<'a, I>(iter: &mut iter::Peekable<I>) -> Option<MetaItemInner>
-    where
-        I: Iterator<Item = &'a TokenTree>,
-    {
+    fn from_tokens(iter: &mut TokenStreamIter<'_>) -> Option<MetaItemInner> {
         match iter.peek() {
             Some(TokenTree::Token(token, _)) if let Some(lit) = MetaItemLit::from_token(token) => {
                 iter.next();
                 return Some(MetaItemInner::Lit(lit));
             }
             Some(TokenTree::Delimited(.., Delimiter::Invisible(_), inner_tokens)) => {
                 iter.next();
-                return MetaItemInner::from_tokens(&mut inner_tokens.iter().peekable());
+                return MetaItemInner::from_tokens(&mut inner_tokens.iter());
             }
             _ => {}
         }

compiler/rustc_ast/src/tokenstream.rs

@@ -676,7 +676,10 @@ impl<'t> TokenStreamIter<'t> {
         TokenStreamIter { stream, index: 0 }
     }
 
-    pub fn peek(&self) -> Option<&TokenTree> {
+    // Peeking could be done via `Peekable`, but most iterators need peeking,
+    // and this is simple and avoids the need to use `peekable` and `Peekable`
+    // at all the use sites.
+    pub fn peek(&self) -> Option<&'t TokenTree> {
         self.stream.0.get(self.index)
     }
 }

compiler/rustc_expand/src/mbe/quoted.rs

@@ -1,4 +1,5 @@
 use rustc_ast::token::{self, Delimiter, IdentIsRaw, NonterminalKind, Token};
+use rustc_ast::tokenstream::TokenStreamIter;
 use rustc_ast::{NodeId, tokenstream};
 use rustc_ast_pretty::pprust;
 use rustc_feature::Features;
@@ -49,7 +50,7 @@ pub(super) fn parse(
 
     // For each token tree in `input`, parse the token into a `self::TokenTree`, consuming
     // additional trees if need be.
-    let mut iter = input.iter().peekable();
+    let mut iter = input.iter();
     while let Some(tree) = iter.next() {
         // Given the parsed tree, if there is a metavar and we are expecting matchers, actually
         // parse out the matcher (i.e., in `$id:ident` this would parse the `:` and `ident`).
@@ -150,7 +151,7 @@ fn maybe_emit_macro_metavar_expr_concat_feature(features: &Features, sess: &Sess
 /// - `features`: language features so we can do feature gating.
 fn parse_tree<'a>(
     tree: &'a tokenstream::TokenTree,
-    outer_iter: &mut impl Iterator<Item = &'a tokenstream::TokenTree>,
+    outer_iter: &mut TokenStreamIter<'a>,
     parsing_patterns: bool,
     sess: &Session,
     node_id: NodeId,
@@ -164,14 +165,15 @@ fn parse_tree<'a>(
             // FIXME: Handle `Invisible`-delimited groups in a more systematic way
             // during parsing.
             let mut next = outer_iter.next();
-            let mut iter: Box<dyn Iterator<Item = &tokenstream::TokenTree>>;
-            match next {
+            let mut iter_storage;
+            let mut iter: &mut TokenStreamIter<'_> = match next {
                 Some(tokenstream::TokenTree::Delimited(.., delim, tts)) if delim.skip() => {
-                    iter = Box::new(tts.iter());
-                    next = iter.next();
+                    iter_storage = tts.iter();
+                    next = iter_storage.next();
+                    &mut iter_storage
                 }
-                _ => iter = Box::new(outer_iter),
-            }
+                _ => outer_iter,
+            };
 
             match next {
                 // `tree` is followed by a delimited set of token trees.
@@ -313,8 +315,8 @@ fn kleene_op(token: &Token) -> Option<KleeneOp> {
 /// - Ok(Ok((op, span))) if the next token tree is a KleeneOp
 /// - Ok(Err(tok, span)) if the next token tree is a token but not a KleeneOp
 /// - Err(span) if the next token tree is not a token
-fn parse_kleene_op<'a>(
-    iter: &mut impl Iterator<Item = &'a tokenstream::TokenTree>,
+fn parse_kleene_op(
+    iter: &mut TokenStreamIter<'_>,
     span: Span,
 ) -> Result<Result<(KleeneOp, Span), Token>, Span> {
     match iter.next() {
@@ -338,8 +340,8 @@ fn parse_kleene_op<'a>(
 /// session `sess`. If the next one (or possibly two) tokens in `iter` correspond to a Kleene
 /// operator and separator, then a tuple with `(separator, KleeneOp)` is returned. Otherwise, an
 /// error with the appropriate span is emitted to `sess` and a dummy value is returned.
-fn parse_sep_and_kleene_op<'a>(
-    iter: &mut impl Iterator<Item = &'a tokenstream::TokenTree>,
+fn parse_sep_and_kleene_op(
+    iter: &mut TokenStreamIter<'_>,
     span: Span,
     sess: &Session,
 ) -> (Option<Token>, KleeneToken) {