syntax: Rename Token into TokenKind

Author: petrochenkov

src/librustc/hir/lowering.rs

@@ -67,7 +67,7 @@ use syntax::source_map::CompilerDesugaringKind::IfTemporary;
 use syntax::std_inject;
 use syntax::symbol::{kw, sym, Symbol};
 use syntax::tokenstream::{TokenStream, TokenTree};
-use syntax::parse::token::{self, Token};
+use syntax::parse::token::{self, TokenKind};
 use syntax::visit::{self, Visitor};
 use syntax_pos::{DUMMY_SP, edition, Span};
 
@@ -1337,7 +1337,7 @@ impl<'a> LoweringContext<'a> {
         }
     }
 
-    fn lower_token(&mut self, token: Token, span: Span) -> TokenStream {
+    fn lower_token(&mut self, token: TokenKind, span: Span) -> TokenStream {
         match token {
             token::Interpolated(nt) => {
                 let tts = nt.to_tokenstream(&self.sess.parse_sess, span);

src/librustc/hir/map/def_collector.rs

@@ -6,7 +6,7 @@ use syntax::ast::*;
 use syntax::ext::hygiene::Mark;
 use syntax::visit;
 use syntax::symbol::{kw, sym};
-use syntax::parse::token::{self, Token};
+use syntax::parse::token::{self, TokenKind};
 use syntax_pos::Span;
 
 /// Creates `DefId`s for nodes in the AST.
@@ -325,7 +325,7 @@ impl<'a> visit::Visitor<'a> for DefCollector<'a> {
         }
     }
 
-    fn visit_token(&mut self, t: Token) {
+    fn visit_token(&mut self, t: TokenKind) {
         if let token::Interpolated(nt) = t {
             if let token::NtExpr(ref expr) = *nt {
                 if let ExprKind::Mac(..) = expr.node {

src/librustc/ich/impls_syntax.rs

@@ -307,7 +307,7 @@ impl_stable_hash_for!(struct token::Lit {
 });
 
 fn hash_token<'a, 'gcx, W: StableHasherResult>(
-    token: &token::Token,
+    token: &token::TokenKind,
     hcx: &mut StableHashingContext<'a>,
     hasher: &mut StableHasher<W>,
 ) {

src/librustc_resolve/build_reduced_graph.rs

@@ -34,7 +34,7 @@ use syntax::ext::base::Determinacy::Undetermined;
 use syntax::ext::hygiene::Mark;
 use syntax::ext::tt::macro_rules;
 use syntax::feature_gate::is_builtin_attr;
-use syntax::parse::token::{self, Token};
+use syntax::parse::token::{self, TokenKind};
 use syntax::span_err;
 use syntax::std_inject::injected_crate_name;
 use syntax::symbol::{kw, sym};
@@ -1052,7 +1052,7 @@ impl<'a, 'b> Visitor<'a> for BuildReducedGraphVisitor<'a, 'b> {
         self.resolver.current_module = parent;
     }
 
-    fn visit_token(&mut self, t: Token) {
+    fn visit_token(&mut self, t: TokenKind) {
         if let token::Interpolated(nt) = t {
             if let token::NtExpr(ref expr) = *nt {
                 if let ast::ExprKind::Mac(..) = expr.node {

src/librustc_save_analysis/span_utils.rs

@@ -5,7 +5,7 @@ use crate::generated_code;
 use std::cell::Cell;
 
 use syntax::parse::lexer::{self, StringReader};
-use syntax::parse::token::{self, Token};
+use syntax::parse::token::{self, TokenKind};
 use syntax_pos::*;
 
 #[derive(Clone)]
@@ -56,7 +56,7 @@ impl<'a> SpanUtils<'a> {
         lexer::StringReader::retokenize(&self.sess.parse_sess, span)
     }
 
-    pub fn sub_span_of_token(&self, span: Span, tok: Token) -> Option<Span> {
+    pub fn sub_span_of_token(&self, span: Span, tok: TokenKind) -> Option<Span> {
         let mut toks = self.retokenise_span(span);
         loop {
             let next = toks.real_token();

src/libsyntax/attr/mod.rs

@@ -20,7 +20,7 @@ use crate::source_map::{BytePos, Spanned, dummy_spanned};
 use crate::parse::lexer::comments::{doc_comment_style, strip_doc_comment_decoration};
 use crate::parse::parser::Parser;
 use crate::parse::{self, ParseSess, PResult};
-use crate::parse::token::{self, Token};
+use crate::parse::token::{self, TokenKind};
 use crate::ptr::P;
 use crate::symbol::{sym, Symbol};
 use crate::ThinVec;
@@ -468,7 +468,7 @@ impl MetaItem {
                 idents.push(TokenTree::Token(mod_sep_span, token::ModSep).into());
             }
             idents.push(TokenTree::Token(segment.ident.span,
-                                         Token::from_ast_ident(segment.ident)).into());
+                                         TokenKind::from_ast_ident(segment.ident)).into());
             last_pos = segment.ident.span.hi();
         }
         self.node.tokens(self.span).append_to_tree_and_joint_vec(&mut idents);

src/libsyntax/ext/tt/macro_parser.rs

@@ -78,7 +78,7 @@ use crate::ast::Ident;
 use crate::ext::tt::quoted::{self, TokenTree};
 use crate::parse::{Directory, ParseSess};
 use crate::parse::parser::{Parser, PathStyle};
-use crate::parse::token::{self, DocComment, Nonterminal, Token};
+use crate::parse::token::{self, DocComment, Nonterminal, TokenKind};
 use crate::print::pprust;
 use crate::symbol::{kw, sym, Symbol};
 use crate::tokenstream::{DelimSpan, TokenStream};
@@ -199,7 +199,7 @@ struct MatcherPos<'root, 'tt: 'root> {
     seq_op: Option<quoted::KleeneOp>,
 
     /// The separator if we are in a repetition.
-    sep: Option<Token>,
+    sep: Option<TokenKind>,
 
     /// The "parent" matcher position if we are in a repetition. That is, the matcher position just
     /// before we enter the sequence.
@@ -273,7 +273,7 @@ pub enum ParseResult<T> {
     Success(T),
     /// Arm failed to match. If the second parameter is `token::Eof`, it indicates an unexpected
     /// end of macro invocation. Otherwise, it indicates that no rules expected the given token.
-    Failure(syntax_pos::Span, Token, &'static str),
+    Failure(syntax_pos::Span, TokenKind, &'static str),
     /// Fatal error (malformed macro?). Abort compilation.
     Error(syntax_pos::Span, String),
 }
@@ -417,7 +417,7 @@ fn nameize<I: Iterator<Item = NamedMatch>>(
 
 /// Generates an appropriate parsing failure message. For EOF, this is "unexpected end...". For
 /// other tokens, this is "unexpected token...".
-pub fn parse_failure_msg(tok: Token) -> String {
+pub fn parse_failure_msg(tok: TokenKind) -> String {
     match tok {
         token::Eof => "unexpected end of macro invocation".to_string(),
         _ => format!(
@@ -428,7 +428,7 @@ pub fn parse_failure_msg(tok: Token) -> String {
 }
 
 /// Performs a token equality check, ignoring syntax context (that is, an unhygienic comparison)
-fn token_name_eq(t1: &Token, t2: &Token) -> bool {
+fn token_name_eq(t1: &TokenKind, t2: &TokenKind) -> bool {
     if let (Some((id1, is_raw1)), Some((id2, is_raw2))) = (t1.ident(), t2.ident()) {
         id1.name == id2.name && is_raw1 == is_raw2
     } else if let (Some(id1), Some(id2)) = (t1.lifetime(), t2.lifetime()) {
@@ -466,7 +466,7 @@ fn inner_parse_loop<'root, 'tt>(
     next_items: &mut Vec<MatcherPosHandle<'root, 'tt>>,
     eof_items: &mut SmallVec<[MatcherPosHandle<'root, 'tt>; 1]>,
     bb_items: &mut SmallVec<[MatcherPosHandle<'root, 'tt>; 1]>,
-    token: &Token,
+    token: &TokenKind,
     span: syntax_pos::Span,
 ) -> ParseResult<()> {
     // Pop items from `cur_items` until it is empty.
@@ -807,7 +807,7 @@ pub fn parse(
 
 /// The token is an identifier, but not `_`.
 /// We prohibit passing `_` to macros expecting `ident` for now.
-fn get_macro_ident(token: &Token) -> Option<(Ident, bool)> {
+fn get_macro_ident(token: &TokenKind) -> Option<(Ident, bool)> {
     match *token {
         token::Ident(ident, is_raw) if ident.name != kw::Underscore =>
             Some((ident, is_raw)),
@@ -819,7 +819,7 @@ fn get_macro_ident(token: &Token) -> Option<(Ident, bool)> {
 ///
 /// Returning `false` is a *stability guarantee* that such a matcher will *never* begin with that
 /// token. Be conservative (return true) if not sure.
-fn may_begin_with(name: Symbol, token: &Token) -> bool {
+fn may_begin_with(name: Symbol, token: &TokenKind) -> bool {
     /// Checks whether the non-terminal may contain a single (non-keyword) identifier.
     fn may_be_ident(nt: &token::Nonterminal) -> bool {
         match *nt {

src/libsyntax/ext/tt/macro_rules.rs

@@ -12,7 +12,7 @@ use crate::feature_gate::Features;
 use crate::parse::{Directory, ParseSess};
 use crate::parse::parser::Parser;
 use crate::parse::token::{self, NtTT};
-use crate::parse::token::Token::*;
+use crate::parse::token::TokenKind::*;
 use crate::symbol::{Symbol, kw, sym};
 use crate::tokenstream::{DelimSpan, TokenStream, TokenTree};
 

src/libsyntax/ext/tt/quoted.rs

@@ -23,12 +23,12 @@ pub struct Delimited {
 
 impl Delimited {
     /// Returns the opening delimiter (possibly `NoDelim`).
-    pub fn open_token(&self) -> token::Token {
+    pub fn open_token(&self) -> token::TokenKind {
         token::OpenDelim(self.delim)
     }
 
     /// Returns the closing delimiter (possibly `NoDelim`).
-    pub fn close_token(&self) -> token::Token {
+    pub fn close_token(&self) -> token::TokenKind {
         token::CloseDelim(self.delim)
     }
 
@@ -58,7 +58,7 @@ pub struct SequenceRepetition {
     /// The sequence of token trees
     pub tts: Vec<TokenTree>,
     /// The optional separator
-    pub separator: Option<token::Token>,
+    pub separator: Option<token::TokenKind>,
     /// Whether the sequence can be repeated zero (*), or one or more times (+)
     pub op: KleeneOp,
     /// The number of `Match`s that appear in the sequence (and subsequences)
@@ -81,7 +81,7 @@ pub enum KleeneOp {
 /// are "first-class" token trees. Useful for parsing macros.
 #[derive(Debug, Clone, PartialEq, RustcEncodable, RustcDecodable)]
 pub enum TokenTree {
-    Token(Span, token::Token),
+    Token(Span, token::TokenKind),
     Delimited(DelimSpan, Lrc<Delimited>),
     /// A kleene-style repetition sequence
     Sequence(DelimSpan, Lrc<SequenceRepetition>),
@@ -366,7 +366,7 @@ where
 
 /// Takes a token and returns `Some(KleeneOp)` if the token is `+` `*` or `?`. Otherwise, return
 /// `None`.
-fn kleene_op(token: &token::Token) -> Option<KleeneOp> {
+fn kleene_op(token: &token::TokenKind) -> Option<KleeneOp> {
     match *token {
         token::BinOp(token::Star) => Some(KleeneOp::ZeroOrMore),
         token::BinOp(token::Plus) => Some(KleeneOp::OneOrMore),
@@ -383,7 +383,7 @@ fn kleene_op(token: &token::Token) -> Option<KleeneOp> {
 fn parse_kleene_op<I>(
     input: &mut I,
     span: Span,
-) -> Result<Result<(KleeneOp, Span), (token::Token, Span)>, Span>
+) -> Result<Result<(KleeneOp, Span), (token::TokenKind, Span)>, Span>
 where
     I: Iterator<Item = tokenstream::TokenTree>,
 {
@@ -422,7 +422,7 @@ fn parse_sep_and_kleene_op<I>(
     attrs: &[ast::Attribute],
     edition: Edition,
     macro_node_id: NodeId,
-) -> (Option<token::Token>, KleeneOp)
+) -> (Option<token::TokenKind>, KleeneOp)
 where
     I: Iterator<Item = tokenstream::TokenTree>,
 {
@@ -447,7 +447,7 @@ fn parse_sep_and_kleene_op_2015<I>(
     _features: &Features,
     _attrs: &[ast::Attribute],
     macro_node_id: NodeId,
-) -> (Option<token::Token>, KleeneOp)
+) -> (Option<token::TokenKind>, KleeneOp)
 where
     I: Iterator<Item = tokenstream::TokenTree>,
 {
@@ -565,7 +565,7 @@ fn parse_sep_and_kleene_op_2018<I>(
     sess: &ParseSess,
     _features: &Features,
     _attrs: &[ast::Attribute],
-) -> (Option<token::Token>, KleeneOp)
+) -> (Option<token::TokenKind>, KleeneOp)
 where
     I: Iterator<Item = tokenstream::TokenTree>,
 {

src/libsyntax/ext/tt/transcribe.rs

@@ -4,7 +4,7 @@ use crate::ext::expand::Marker;
 use crate::ext::tt::macro_parser::{MatchedNonterminal, MatchedSeq, NamedMatch};
 use crate::ext::tt::quoted;
 use crate::mut_visit::noop_visit_tt;
-use crate::parse::token::{self, NtTT, Token};
+use crate::parse::token::{self, NtTT, TokenKind};
 use crate::tokenstream::{DelimSpan, TokenStream, TokenTree, TreeAndJoint};
 
 use smallvec::{smallvec, SmallVec};
@@ -18,7 +18,7 @@ use std::rc::Rc;
 /// An iterator over the token trees in a delimited token tree (`{ ... }`) or a sequence (`$(...)`).
 enum Frame {
     Delimited { forest: Lrc<quoted::Delimited>, idx: usize, span: DelimSpan },
-    Sequence { forest: Lrc<quoted::SequenceRepetition>, idx: usize, sep: Option<Token> },
+    Sequence { forest: Lrc<quoted::SequenceRepetition>, idx: usize, sep: Option<TokenKind> },
 }
 
 impl Frame {
@@ -242,7 +242,7 @@ pub fn transcribe(
                         Ident::new(ident.name, ident.span.apply_mark(cx.current_expansion.mark));
                     sp = sp.apply_mark(cx.current_expansion.mark);
                     result.push(TokenTree::Token(sp, token::Dollar).into());
-                    result.push(TokenTree::Token(sp, token::Token::from_ast_ident(ident)).into());
+                    result.push(TokenTree::Token(sp, token::TokenKind::from_ast_ident(ident)).into());
                 }
             }
 

src/libsyntax/mut_visit.rs

@@ -9,7 +9,7 @@
 
 use crate::ast::*;
 use crate::source_map::{Spanned, respan};
-use crate::parse::token::{self, Token};
+use crate::parse::token::{self, TokenKind};
 use crate::ptr::P;
 use crate::ThinVec;
 use crate::tokenstream::*;
@@ -262,7 +262,7 @@ pub trait MutVisitor: Sized {
         noop_visit_tts(tts, self);
     }
 
-    fn visit_token(&mut self, t: &mut Token) {
+    fn visit_token(&mut self, t: &mut TokenKind) {
         noop_visit_token(t, self);
     }
 
@@ -596,7 +596,7 @@ pub fn noop_visit_tts<T: MutVisitor>(TokenStream(tts): &mut TokenStream, vis: &m
 }
 
 // apply ident visitor if it's an ident, apply other visits to interpolated nodes
-pub fn noop_visit_token<T: MutVisitor>(t: &mut Token, vis: &mut T) {
+pub fn noop_visit_token<T: MutVisitor>(t: &mut TokenKind, vis: &mut T) {
     match t {
         token::Ident(id, _is_raw) => vis.visit_ident(id),
         token::Lifetime(id) => vis.visit_ident(id),

src/libsyntax/parse/diagnostics.rs

@@ -229,8 +229,8 @@ impl<'a> Parser<'a> {
 
     pub fn expected_one_of_not_found(
         &mut self,
-        edible: &[token::Token],
-        inedible: &[token::Token],
+        edible: &[token::TokenKind],
+        inedible: &[token::TokenKind],
     ) -> PResult<'a, bool /* recovered */> {
         fn tokens_to_string(tokens: &[TokenType]) -> String {
             let mut i = tokens.iter();
@@ -368,7 +368,7 @@ impl<'a> Parser<'a> {
 
     /// Eats and discards tokens until one of `kets` is encountered. Respects token trees,
     /// passes through any errors encountered. Used for error recovery.
-    crate fn eat_to_tokens(&mut self, kets: &[&token::Token]) {
+    crate fn eat_to_tokens(&mut self, kets: &[&token::TokenKind]) {
         let handler = self.diagnostic();
 
         if let Err(ref mut err) = self.parse_seq_to_before_tokens(
@@ -388,7 +388,7 @@ impl<'a> Parser<'a> {
     /// let _ = vec![1, 2, 3].into_iter().collect::<Vec<usize>>>>();
     ///                                                        ^^ help: remove extra angle brackets
     /// ```
-    crate fn check_trailing_angle_brackets(&mut self, segment: &PathSegment, end: token::Token) {
+    crate fn check_trailing_angle_brackets(&mut self, segment: &PathSegment, end: token::TokenKind) {
         // This function is intended to be invoked after parsing a path segment where there are two
         // cases:
         //
@@ -726,7 +726,7 @@ impl<'a> Parser<'a> {
     /// closing delimiter.
     pub fn unexpected_try_recover(
         &mut self,
-        t: &token::Token,
+        t: &token::TokenKind,
     ) -> PResult<'a, bool /* recovered */> {
         let token_str = pprust::token_to_string(t);
         let this_token_str = self.this_token_descr();
@@ -903,7 +903,7 @@ impl<'a> Parser<'a> {
 
     crate fn recover_closing_delimiter(
         &mut self,
-        tokens: &[token::Token],
+        tokens: &[token::TokenKind],
         mut err: DiagnosticBuilder<'a>,
     ) -> PResult<'a, bool> {
         let mut pos = None;