Skip to content

Commit e0127db

Browse files
petrochenkovpetrochenkov
committed
syntax: Use Token in TokenTree::Token 
1 parent a3425ed commit e0127db

31 files changed

+327
-312
lines changed

‎src/librustc/hir/lowering.rs

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -67,7 +67,7 @@ use syntax::source_map::CompilerDesugaringKind::IfTemporary;
6767
use syntax::std_inject;
6868
use syntax::symbol::{kw, sym, Symbol};
6969
use syntax::tokenstream::{TokenStream, TokenTree};
70-
use syntax::parse::token::{self, TokenKind};
70+
use syntax::parse::token::{self, Token};
7171
use syntax::visit::{self, Visitor};
7272
use syntax_pos::{DUMMY_SP, edition, Span};
7373

@@ -1328,7 +1328,7 @@ impl<'a> LoweringContext<'a> {
13281328

13291329
fn lower_token_tree(&mut self, tree: TokenTree) -> TokenStream {
13301330
match tree {
1331-
TokenTree::Token(span, token) => self.lower_token(token, span),
1331+
TokenTree::Token(token) => self.lower_token(token),
13321332
TokenTree::Delimited(span, delim, tts) => TokenTree::Delimited(
13331333
span,
13341334
delim,
@@ -1337,13 +1337,13 @@ impl<'a> LoweringContext<'a> {
13371337
}
13381338
}
13391339

1340-
fn lower_token(&mut self, token: TokenKind, span: Span) -> TokenStream {
1341-
match token {
1340+
fn lower_token(&mut self, token: Token) -> TokenStream {
1341+
match token.kind {
13421342
token::Interpolated(nt) => {
1343-
let tts = nt.to_tokenstream(&self.sess.parse_sess, span);
1343+
let tts = nt.to_tokenstream(&self.sess.parse_sess, token.span);
13441344
self.lower_token_stream(tts)
13451345
}
1346-
other => TokenTree::Token(span, other).into(),
1346+
_ => TokenTree::Token(token).into(),
13471347
}
13481348
}
13491349

‎src/librustc/ich/impls_syntax.rs

Lines changed: 64 additions & 60 deletions
Original file line numberDiff line numberDiff line change
@@ -261,9 +261,8 @@ for tokenstream::TokenTree {
261261
hasher: &mut StableHasher<W>) {
262262
mem::discriminant(self).hash_stable(hcx, hasher);
263263
match *self {
264-
tokenstream::TokenTree::Token(span, ref token) => {
265-
span.hash_stable(hcx, hasher);
266-
hash_token(token, hcx, hasher);
264+
tokenstream::TokenTree::Token(ref token) => {
265+
token.hash_stable(hcx, hasher);
267266
}
268267
tokenstream::TokenTree::Delimited(span, delim, ref tts) => {
269268
span.hash_stable(hcx, hasher);
@@ -306,70 +305,75 @@ impl_stable_hash_for!(struct token::Lit {
306305
suffix
307306
});
308307

309-
fn hash_token<'a, 'gcx, W: StableHasherResult>(
310-
token: &token::TokenKind,
311-
hcx: &mut StableHashingContext<'a>,
312-
hasher: &mut StableHasher<W>,
313-
) {
314-
mem::discriminant(token).hash_stable(hcx, hasher);
315-
match *token {
316-
token::Eq |
317-
token::Lt |
318-
token::Le |
319-
token::EqEq |
320-
token::Ne |
321-
token::Ge |
322-
token::Gt |
323-
token::AndAnd |
324-
token::OrOr |
325-
token::Not |
326-
token::Tilde |
327-
token::At |
328-
token::Dot |
329-
token::DotDot |
330-
token::DotDotDot |
331-
token::DotDotEq |
332-
token::Comma |
333-
token::Semi |
334-
token::Colon |
335-
token::ModSep |
336-
token::RArrow |
337-
token::LArrow |
338-
token::FatArrow |
339-
token::Pound |
340-
token::Dollar |
341-
token::Question |
342-
token::SingleQuote |
343-
token::Whitespace |
344-
token::Comment |
345-
token::Eof => {}
346-
347-
token::BinOp(bin_op_token) |
348-
token::BinOpEq(bin_op_token) => {
349-
std_hash::Hash::hash(&bin_op_token, hasher);
350-
}
308+
impl<'a> HashStable<StableHashingContext<'a>> for token::TokenKind {
309+
fn hash_stable<W: StableHasherResult>(&self,
310+
hcx: &mut StableHashingContext<'a>,
311+
hasher: &mut StableHasher<W>) {
312+
mem::discriminant(self).hash_stable(hcx, hasher);
313+
match *self {
314+
token::Eq |
315+
token::Lt |
316+
token::Le |
317+
token::EqEq |
318+
token::Ne |
319+
token::Ge |
320+
token::Gt |
321+
token::AndAnd |
322+
token::OrOr |
323+
token::Not |
324+
token::Tilde |
325+
token::At |
326+
token::Dot |
327+
token::DotDot |
328+
token::DotDotDot |
329+
token::DotDotEq |
330+
token::Comma |
331+
token::Semi |
332+
token::Colon |
333+
token::ModSep |
334+
token::RArrow |
335+
token::LArrow |
336+
token::FatArrow |
337+
token::Pound |
338+
token::Dollar |
339+
token::Question |
340+
token::SingleQuote |
341+
token::Whitespace |
342+
token::Comment |
343+
token::Eof => {}
344+
345+
token::BinOp(bin_op_token) |
346+
token::BinOpEq(bin_op_token) => {
347+
std_hash::Hash::hash(&bin_op_token, hasher);
348+
}
351349

352-
token::OpenDelim(delim_token) |
353-
token::CloseDelim(delim_token) => {
354-
std_hash::Hash::hash(&delim_token, hasher);
355-
}
356-
token::Literal(lit) => lit.hash_stable(hcx, hasher),
350+
token::OpenDelim(delim_token) |
351+
token::CloseDelim(delim_token) => {
352+
std_hash::Hash::hash(&delim_token, hasher);
353+
}
354+
token::Literal(lit) => lit.hash_stable(hcx, hasher),
357355

358-
token::Ident(ident, is_raw) => {
359-
ident.name.hash_stable(hcx, hasher);
360-
is_raw.hash_stable(hcx, hasher);
361-
}
362-
token::Lifetime(ident) => ident.name.hash_stable(hcx, hasher),
356+
token::Ident(ident, is_raw) => {
357+
ident.name.hash_stable(hcx, hasher);
358+
is_raw.hash_stable(hcx, hasher);
359+
}
360+
token::Lifetime(ident) => ident.name.hash_stable(hcx, hasher),
363361

364-
token::Interpolated(_) => {
365-
bug!("interpolated tokens should not be present in the HIR")
366-
}
362+
token::Interpolated(_) => {
363+
bug!("interpolated tokens should not be present in the HIR")
364+
}
367365

368-
token::DocComment(val) |
369-
token::Shebang(val) => val.hash_stable(hcx, hasher),
366+
token::DocComment(val) |
367+
token::Shebang(val) => val.hash_stable(hcx, hasher),
368+
}
370369
}
371370
}
372371

372+
impl_stable_hash_for!(struct token::Token {
373+
kind,
374+
span
375+
});
376+
373377
impl_stable_hash_for!(enum ::syntax::ast::NestedMetaItem {
374378
MetaItem(meta_item),
375379
Literal(lit)

‎src/librustc_lint/builtin.rs

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1414,11 +1414,11 @@ impl KeywordIdents {
14141414
fn check_tokens(&mut self, cx: &EarlyContext<'_>, tokens: TokenStream) {
14151415
for tt in tokens.into_trees() {
14161416
match tt {
1417-
TokenTree::Token(span, tok) => match tok.ident() {
1417+
TokenTree::Token(token) => match token.ident() {
14181418
// only report non-raw idents
14191419
Some((ident, false)) => {
14201420
self.check_ident_token(cx, UnderMacro(true), ast::Ident {
1421-
span: span.substitute_dummy(ident.span),
1421+
span: token.span.substitute_dummy(ident.span),
14221422
..ident
14231423
});
14241424
}

‎src/librustdoc/html/highlight.rs

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -234,7 +234,7 @@ impl<'a> Classifier<'a> {
234234
// reference or dereference operator or a reference or pointer type, instead of the
235235
// bit-and or multiplication operator.
236236
token::BinOp(token::And) | token::BinOp(token::Star)
237-
if self.lexer.peek().kind != token::Whitespace => Class::RefKeyWord,
237+
if self.lexer.peek() != token::Whitespace => Class::RefKeyWord,
238238

239239
// Consider this as part of a macro invocation if there was a
240240
// leading identifier.
@@ -335,7 +335,7 @@ impl<'a> Classifier<'a> {
335335
sym::Option | sym::Result => Class::PreludeTy,
336336
sym::Some | sym::None | sym::Ok | sym::Err => Class::PreludeVal,
337337

338-
_ if token.kind.is_reserved_ident() => Class::KeyWord,
338+
_ if token.is_reserved_ident() => Class::KeyWord,
339339

340340
_ => {
341341
if self.in_macro_nonterminal {

‎src/libsyntax/attr/mod.rs

Lines changed: 18 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,7 @@ use crate::source_map::{BytePos, Spanned, dummy_spanned};
2020
use crate::parse::lexer::comments::{doc_comment_style, strip_doc_comment_decoration};
2121
use crate::parse::parser::Parser;
2222
use crate::parse::{self, ParseSess, PResult};
23-
use crate::parse::token::{self, TokenKind};
23+
use crate::parse::token::{self, Token, TokenKind};
2424
use crate::ptr::P;
2525
use crate::symbol::{sym, Symbol};
2626
use crate::ThinVec;
@@ -465,9 +465,9 @@ impl MetaItem {
465465
let mod_sep_span = Span::new(last_pos,
466466
segment.ident.span.lo(),
467467
segment.ident.span.ctxt());
468-
idents.push(TokenTree::Token(mod_sep_span, token::ModSep).into());
468+
idents.push(TokenTree::token(mod_sep_span, token::ModSep).into());
469469
}
470-
idents.push(TokenTree::Token(segment.ident.span,
470+
idents.push(TokenTree::token(segment.ident.span,
471471
TokenKind::from_ast_ident(segment.ident)).into());
472472
last_pos = segment.ident.span.hi();
473473
}
@@ -480,10 +480,10 @@ impl MetaItem {
480480
{
481481
// FIXME: Share code with `parse_path`.
482482
let path = match tokens.next() {
483-
Some(TokenTree::Token(span, token @ token::Ident(..))) |
484-
Some(TokenTree::Token(span, token @ token::ModSep)) => 'arm: {
485-
let mut segments = if let token::Ident(ident, _) = token {
486-
if let Some(TokenTree::Token(_, token::ModSep)) = tokens.peek() {
483+
Some(TokenTree::Token(Token { kind: kind @ token::Ident(..), span })) |
484+
Some(TokenTree::Token(Token { kind: kind @ token::ModSep, span })) => 'arm: {
485+
let mut segments = if let token::Ident(ident, _) = kind {
486+
if let Some(TokenTree::Token(Token { kind: token::ModSep, .. })) = tokens.peek() {
487487
tokens.next();
488488
vec![PathSegment::from_ident(ident.with_span_pos(span))]
489489
} else {
@@ -493,13 +493,12 @@ impl MetaItem {
493493
vec![PathSegment::path_root(span)]
494494
};
495495
loop {
496-
if let Some(TokenTree::Token(span,
497-
token::Ident(ident, _))) = tokens.next() {
496+
if let Some(TokenTree::Token(Token { kind: token::Ident(ident, _), span })) = tokens.next() {
498497
segments.push(PathSegment::from_ident(ident.with_span_pos(span)));
499498
} else {
500499
return None;
501500
}
502-
if let Some(TokenTree::Token(_, token::ModSep)) = tokens.peek() {
501+
if let Some(TokenTree::Token(Token { kind: token::ModSep, .. })) = tokens.peek() {
503502
tokens.next();
504503
} else {
505504
break;
@@ -508,7 +507,7 @@ impl MetaItem {
508507
let span = span.with_hi(segments.last().unwrap().ident.span.hi());
509508
Path { span, segments }
510509
}
511-
Some(TokenTree::Token(_, token::Interpolated(nt))) => match *nt {
510+
Some(TokenTree::Token(Token { kind: token::Interpolated(nt), .. })) => match *nt {
512511
token::Nonterminal::NtIdent(ident, _) => Path::from_ident(ident),
513512
token::Nonterminal::NtMeta(ref meta) => return Some(meta.clone()),
514513
token::Nonterminal::NtPath(ref path) => path.clone(),
@@ -533,15 +532,15 @@ impl MetaItemKind {
533532
match *self {
534533
MetaItemKind::Word => TokenStream::empty(),
535534
MetaItemKind::NameValue(ref lit) => {
536-
let mut vec = vec![TokenTree::Token(span, token::Eq).into()];
535+
let mut vec = vec![TokenTree::token(span, token::Eq).into()];
537536
lit.tokens().append_to_tree_and_joint_vec(&mut vec);
538537
TokenStream::new(vec)
539538
}
540539
MetaItemKind::List(ref list) => {
541540
let mut tokens = Vec::new();
542541
for (i, item) in list.iter().enumerate() {
543542
if i > 0 {
544-
tokens.push(TokenTree::Token(span, token::Comma).into());
543+
tokens.push(TokenTree::token(span, token::Comma).into());
545544
}
546545
item.tokens().append_to_tree_and_joint_vec(&mut tokens);
547546
}
@@ -558,10 +557,10 @@ impl MetaItemKind {
558557
where I: Iterator<Item = TokenTree>,
559558
{
560559
let delimited = match tokens.peek().cloned() {
561-
Some(TokenTree::Token(_, token::Eq)) => {
560+
Some(TokenTree::Token(token)) if token == token::Eq => {
562561
tokens.next();
563-
return if let Some(TokenTree::Token(span, token)) = tokens.next() {
564-
Lit::from_token(&token, span).ok().map(MetaItemKind::NameValue)
562+
return if let Some(TokenTree::Token(token)) = tokens.next() {
563+
Lit::from_token(&token, token.span).ok().map(MetaItemKind::NameValue)
565564
} else {
566565
None
567566
};
@@ -579,7 +578,7 @@ impl MetaItemKind {
579578
let item = NestedMetaItem::from_tokens(&mut tokens)?;
580579
result.push(item);
581580
match tokens.next() {
582-
None | Some(TokenTree::Token(_, token::Comma)) => {}
581+
None | Some(TokenTree::Token(Token { kind: token::Comma, .. })) => {}
583582
_ => return None,
584583
}
585584
}
@@ -605,8 +604,8 @@ impl NestedMetaItem {
605604
fn from_tokens<I>(tokens: &mut iter::Peekable<I>) -> Option<NestedMetaItem>
606605
where I: Iterator<Item = TokenTree>,
607606
{
608-
if let Some(TokenTree::Token(span, token)) = tokens.peek().cloned() {
609-
if let Ok(lit) = Lit::from_token(&token, span) {
607+
if let Some(TokenTree::Token(token)) = tokens.peek().cloned() {
608+
if let Ok(lit) = Lit::from_token(&token, token.span) {
610609
tokens.next();
611610
return Some(NestedMetaItem::Literal(lit));
612611
}

‎src/libsyntax/diagnostics/plugin.rs

Lines changed: 8 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@ use crate::ast::{self, Ident, Name};
55
use crate::source_map;
66
use crate::ext::base::{ExtCtxt, MacEager, MacResult};
77
use crate::ext::build::AstBuilder;
8-
use crate::parse::token;
8+
use crate::parse::token::{self, Token};
99
use crate::ptr::P;
1010
use crate::symbol::kw;
1111
use crate::tokenstream::{TokenTree};
@@ -34,7 +34,7 @@ pub fn expand_diagnostic_used<'cx>(ecx: &'cx mut ExtCtxt<'_>,
3434
token_tree: &[TokenTree])
3535
-> Box<dyn MacResult+'cx> {
3636
let code = match (token_tree.len(), token_tree.get(0)) {
37-
(1, Some(&TokenTree::Token(_, token::Ident(code, _)))) => code,
37+
(1, Some(&TokenTree::Token(Token { kind: token::Ident(code, _), .. }))) => code,
3838
_ => unreachable!()
3939
};
4040

@@ -72,12 +72,12 @@ pub fn expand_register_diagnostic<'cx>(ecx: &'cx mut ExtCtxt<'_>,
7272
token_tree.get(1),
7373
token_tree.get(2)
7474
) {
75-
(1, Some(&TokenTree::Token(_, token::Ident(ref code, _))), None, None) => {
75+
(1, Some(&TokenTree::Token(Token { kind: token::Ident(ref code, _), .. })), None, None) => {
7676
(code, None)
7777
},
78-
(3, Some(&TokenTree::Token(_, token::Ident(ref code, _))),
79-
Some(&TokenTree::Token(_, token::Comma)),
80-
Some(&TokenTree::Token(_, token::Literal(token::Lit { symbol, .. })))) => {
78+
(3, Some(&TokenTree::Token(Token { kind: token::Ident(ref code, _), .. })),
79+
Some(&TokenTree::Token(Token { kind: token::Comma, .. })),
80+
Some(&TokenTree::Token(Token { kind: token::Literal(token::Lit { symbol, .. }), .. }))) => {
8181
(code, Some(symbol))
8282
}
8383
_ => unreachable!()
@@ -143,9 +143,9 @@ pub fn expand_build_diagnostic_array<'cx>(ecx: &'cx mut ExtCtxt<'_>,
143143
let (crate_name, name) = match (&token_tree[0], &token_tree[2]) {
144144
(
145145
// Crate name.
146-
&TokenTree::Token(_, token::Ident(ref crate_name, _)),
146+
&TokenTree::Token(Token { kind: token::Ident(ref crate_name, _), .. }),
147147
// DIAGNOSTICS ident.
148-
&TokenTree::Token(_, token::Ident(ref name, _))
148+
&TokenTree::Token(Token { kind: token::Ident(ref name, _), .. })
149149
) => (*&crate_name, name),
150150
_ => unreachable!()
151151
};

‎src/libsyntax/ext/base.rs

Lines changed: 6 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -265,10 +265,12 @@ impl<F> TTMacroExpander for F
265265

266266
impl MutVisitor for AvoidInterpolatedIdents {
267267
fn visit_tt(&mut self, tt: &mut tokenstream::TokenTree) {
268-
if let tokenstream::TokenTree::Token(_, token::Interpolated(nt)) = tt {
269-
if let token::NtIdent(ident, is_raw) = **nt {
270-
*tt = tokenstream::TokenTree::Token(ident.span,
271-
token::Ident(ident, is_raw));
268+
if let tokenstream::TokenTree::Token(token) = tt {
269+
if let token::Interpolated(nt) = &token.kind {
270+
if let token::NtIdent(ident, is_raw) = **nt {
271+
*tt = tokenstream::TokenTree::token(ident.span,
272+
token::Ident(ident, is_raw));
273+
}
272274
}
273275
}
274276
mut_visit::noop_visit_tt(tt, self)

‎src/libsyntax/ext/expand.rs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -585,7 +585,7 @@ impl<'a, 'b> MacroExpander<'a, 'b> {
585585
}
586586
AttrProcMacro(ref mac, ..) => {
587587
self.gate_proc_macro_attr_item(attr.span, &item);
588-
let item_tok = TokenTree::Token(DUMMY_SP, token::Interpolated(Lrc::new(match item {
588+
let item_tok = TokenTree::token(DUMMY_SP, token::Interpolated(Lrc::new(match item {
589589
Annotatable::Item(item) => token::NtItem(item),
590590
Annotatable::TraitItem(item) => token::NtTraitItem(item.into_inner()),
591591
Annotatable::ImplItem(item) => token::NtImplItem(item.into_inner()),

0 commit comments

Comments
 (0)