Skip to content

Commit 8739668

Browse files
committed
Simplify conversions between tokens and semantic literals
1 parent a5b3f33 commit 8739668

File tree

10 files changed

+259
-324
lines changed

10 files changed

+259
-324
lines changed

src/librustc/hir/print.rs

Lines changed: 3 additions & 53 deletions
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@ use syntax::parse::ParseSess;
55
use syntax::parse::lexer::comments;
66
use syntax::print::pp::{self, Breaks};
77
use syntax::print::pp::Breaks::{Consistent, Inconsistent};
8-
use syntax::print::pprust::PrintState;
8+
use syntax::print::pprust::{self, PrintState};
99
use syntax::ptr::P;
1010
use syntax::symbol::keywords;
1111
use syntax::util::parser::{self, AssocOp, Fixity};
@@ -15,7 +15,6 @@ use crate::hir;
1515
use crate::hir::{PatKind, GenericBound, TraitBoundModifier, RangeEnd};
1616
use crate::hir::{GenericParam, GenericParamKind, GenericArg};
1717

18-
use std::ascii;
1918
use std::borrow::Cow;
2019
use std::cell::Cell;
2120
use std::io::{self, Write, Read};
@@ -1251,57 +1250,8 @@ impl<'a> State<'a> {
12511250

12521251
fn print_literal(&mut self, lit: &hir::Lit) -> io::Result<()> {
12531252
self.maybe_print_comment(lit.span.lo())?;
1254-
match lit.node {
1255-
hir::LitKind::Str(st, style) => self.print_string(&st.as_str(), style),
1256-
hir::LitKind::Err(st) => {
1257-
let st = st.as_str().escape_debug().to_string();
1258-
let mut res = String::with_capacity(st.len() + 2);
1259-
res.push('\'');
1260-
res.push_str(&st);
1261-
res.push('\'');
1262-
self.writer().word(res)
1263-
}
1264-
hir::LitKind::Byte(byte) => {
1265-
let mut res = String::from("b'");
1266-
res.extend(ascii::escape_default(byte).map(|c| c as char));
1267-
res.push('\'');
1268-
self.writer().word(res)
1269-
}
1270-
hir::LitKind::Char(ch) => {
1271-
let mut res = String::from("'");
1272-
res.extend(ch.escape_default());
1273-
res.push('\'');
1274-
self.writer().word(res)
1275-
}
1276-
hir::LitKind::Int(i, t) => {
1277-
match t {
1278-
ast::LitIntType::Signed(st) => {
1279-
self.writer().word(st.val_to_string(i as i128))
1280-
}
1281-
ast::LitIntType::Unsigned(ut) => {
1282-
self.writer().word(ut.val_to_string(i))
1283-
}
1284-
ast::LitIntType::Unsuffixed => {
1285-
self.writer().word(i.to_string())
1286-
}
1287-
}
1288-
}
1289-
hir::LitKind::Float(ref f, t) => {
1290-
self.writer().word(format!("{}{}", &f, t.ty_to_string()))
1291-
}
1292-
hir::LitKind::FloatUnsuffixed(ref f) => self.writer().word(f.as_str().to_string()),
1293-
hir::LitKind::Bool(val) => {
1294-
if val { self.writer().word("true") } else { self.writer().word("false") }
1295-
}
1296-
hir::LitKind::ByteStr(ref v) => {
1297-
let mut escaped: String = String::new();
1298-
for &ch in v.iter() {
1299-
escaped.extend(ascii::escape_default(ch)
1300-
.map(|c| c as char));
1301-
}
1302-
self.writer().word(format!("b\"{}\"", escaped))
1303-
}
1304-
}
1253+
let (token, suffix) = lit.node.to_lit_token();
1254+
self.writer().word(pprust::literal_to_string(token, suffix))
13051255
}
13061256

13071257
pub fn print_expr(&mut self, expr: &hir::Expr) -> io::Result<()> {

src/librustdoc/clean/cfg.rs

Lines changed: 10 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -591,13 +591,10 @@ mod test {
591591
let mi = dummy_meta_item_word("all");
592592
assert_eq!(Cfg::parse(&mi), Ok(word_cfg("all")));
593593

594-
let node = LitKind::Str(Symbol::intern("done"), StrStyle::Cooked);
595-
let (token, suffix) = node.lit_token();
596-
let mi = MetaItem {
597-
path: Path::from_ident(Ident::from_str("all")),
598-
node: MetaItemKind::NameValue(Lit { node, token, suffix, span: DUMMY_SP }),
599-
span: DUMMY_SP,
600-
};
594+
let mi = attr::mk_name_value_item_str(
595+
Ident::from_str("all"),
596+
dummy_spanned(Symbol::intern("done"))
597+
);
601598
assert_eq!(Cfg::parse(&mi), Ok(name_value_cfg("all", "done")));
602599

603600
let mi = dummy_meta_item_list!(all, [a, b]);
@@ -625,13 +622,12 @@ mod test {
625622
#[test]
626623
fn test_parse_err() {
627624
with_globals(|| {
628-
let node = LitKind::Bool(false);
629-
let (token, suffix) = node.lit_token();
630-
let mi = MetaItem {
631-
path: Path::from_ident(Ident::from_str("foo")),
632-
node: MetaItemKind::NameValue(Lit { node, token, suffix, span: DUMMY_SP }),
633-
span: DUMMY_SP,
634-
};
625+
let mi = attr::mk_name_value_item(
626+
DUMMY_SP,
627+
Ident::from_str("foo"),
628+
LitKind::Bool(false),
629+
DUMMY_SP,
630+
);
635631
assert!(Cfg::parse(&mi).is_err());
636632

637633
let mi = dummy_meta_item_list!(not, [a, b]);

src/libsyntax/attr/mod.rs

Lines changed: 58 additions & 46 deletions
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,7 @@ pub use StabilityLevel::*;
1414
use crate::ast;
1515
use crate::ast::{AttrId, Attribute, AttrStyle, Name, Ident, Path, PathSegment};
1616
use crate::ast::{MetaItem, MetaItemKind, NestedMetaItem};
17-
use crate::ast::{Lit, LitKind, Expr, ExprKind, Item, Local, Stmt, StmtKind, GenericParam};
17+
use crate::ast::{Lit, LitKind, Expr, Item, Local, Stmt, StmtKind, GenericParam};
1818
use crate::mut_visit::visit_clobber;
1919
use crate::source_map::{BytePos, Spanned, dummy_spanned};
2020
use crate::parse::lexer::comments::{doc_comment_style, strip_doc_comment_decoration};
@@ -27,9 +27,11 @@ use crate::ThinVec;
2727
use crate::tokenstream::{TokenStream, TokenTree, DelimSpan};
2828
use crate::GLOBALS;
2929

30+
use errors::Handler;
3031
use log::debug;
3132
use syntax_pos::{FileName, Span};
3233

34+
use std::ascii;
3335
use std::iter;
3436
use std::ops::DerefMut;
3537

@@ -350,14 +352,13 @@ impl Attribute {
350352
/* Constructors */
351353

352354
pub fn mk_name_value_item_str(ident: Ident, value: Spanned<Symbol>) -> MetaItem {
353-
let node = LitKind::Str(value.node, ast::StrStyle::Cooked);
354-
let (token, suffix) = node.lit_token();
355-
let value = Lit { node, token, suffix, span: value.span };
356-
mk_name_value_item(ident.span.to(value.span), ident, value)
355+
let lit_kind = LitKind::Str(value.node, ast::StrStyle::Cooked);
356+
mk_name_value_item(ident.span.to(value.span), ident, lit_kind, value.span)
357357
}
358358

359-
pub fn mk_name_value_item(span: Span, ident: Ident, value: Lit) -> MetaItem {
360-
MetaItem { path: Path::from_ident(ident), span, node: MetaItemKind::NameValue(value) }
359+
pub fn mk_name_value_item(span: Span, ident: Ident, lit_kind: LitKind, lit_span: Span) -> MetaItem {
360+
let lit = Lit::from_lit_kind(lit_kind, lit_span);
361+
MetaItem { path: Path::from_ident(ident), span, node: MetaItemKind::NameValue(lit) }
361362
}
362363

363364
pub fn mk_list_item(span: Span, ident: Ident, items: Vec<NestedMetaItem>) -> MetaItem {
@@ -419,9 +420,8 @@ pub fn mk_spanned_attr_outer(sp: Span, id: AttrId, item: MetaItem) -> Attribute
419420

420421
pub fn mk_sugared_doc_attr(id: AttrId, text: Symbol, span: Span) -> Attribute {
421422
let style = doc_comment_style(&text.as_str());
422-
let node = LitKind::Str(text, ast::StrStyle::Cooked);
423-
let (token, suffix) = node.lit_token();
424-
let lit = Lit { node, token, suffix, span };
423+
let lit_kind = LitKind::Str(text, ast::StrStyle::Cooked);
424+
let lit = Lit::from_lit_kind(lit_kind, span);
425425
Attribute {
426426
id,
427427
style,
@@ -565,9 +565,7 @@ impl MetaItemKind {
565565
Some(TokenTree::Token(_, token::Eq)) => {
566566
tokens.next();
567567
return if let Some(TokenTree::Token(span, token)) = tokens.next() {
568-
LitKind::from_token(token).map(|(node, token, suffix)| {
569-
MetaItemKind::NameValue(Lit { node, token, suffix, span })
570-
})
568+
Lit::from_token(&token, span, None).map(MetaItemKind::NameValue)
571569
} else {
572570
None
573571
};
@@ -612,9 +610,9 @@ impl NestedMetaItem {
612610
where I: Iterator<Item = TokenTree>,
613611
{
614612
if let Some(TokenTree::Token(span, token)) = tokens.peek().cloned() {
615-
if let Some((node, token, suffix)) = LitKind::from_token(token) {
613+
if let Some(lit) = Lit::from_token(&token, span, None) {
616614
tokens.next();
617-
return Some(NestedMetaItem::Literal(Lit { node, token, suffix, span }));
615+
return Some(NestedMetaItem::Literal(lit));
618616
}
619617
}
620618

@@ -624,21 +622,19 @@ impl NestedMetaItem {
624622

625623
impl Lit {
626624
crate fn tokens(&self) -> TokenStream {
627-
TokenTree::Token(self.span, self.node.token()).into()
625+
let token = match self.token {
626+
token::Bool(symbol) => Token::Ident(Ident::with_empty_ctxt(symbol), false),
627+
token => Token::Literal(token, self.suffix),
628+
};
629+
TokenTree::Token(self.span, token).into()
628630
}
629631
}
630632

631633
impl LitKind {
632-
fn token(&self) -> Token {
633-
match self.lit_token() {
634-
(token::Bool(symbol), _) => Token::Ident(Ident::with_empty_ctxt(symbol), false),
635-
(lit, suffix) => Token::Literal(lit, suffix),
636-
}
637-
}
638-
639-
pub fn lit_token(&self) -> (token::Lit, Option<Symbol>) {
640-
use std::ascii;
641-
634+
/// Attempts to recover a token from semantic literal.
635+
/// This function is used when the original token doesn't exist (e.g. the literal is created
636+
/// by an AST-based macro) or unavailable (e.g. from HIR pretty-printing).
637+
pub fn to_lit_token(&self) -> (token::Lit, Option<Symbol>) {
642638
match *self {
643639
LitKind::Str(string, ast::StrStyle::Cooked) => {
644640
let escaped = string.as_str().escape_default().to_string();
@@ -679,29 +675,45 @@ impl LitKind {
679675
LitKind::Err(val) => (token::Lit::Err(val), None),
680676
}
681677
}
678+
}
682679

683-
fn from_token(token: Token) -> Option<(LitKind, token::Lit, Option<Symbol>)> {
684-
match token {
685-
Token::Ident(ident, false) if ident.name == keywords::True.name() =>
686-
Some((LitKind::Bool(true), token::Bool(ident.name), None)),
687-
Token::Ident(ident, false) if ident.name == keywords::False.name() =>
688-
Some((LitKind::Bool(false), token::Bool(ident.name), None)),
689-
Token::Interpolated(nt) => match *nt {
690-
token::NtExpr(ref v) | token::NtLiteral(ref v) => match v.node {
691-
ExprKind::Lit(ref lit) => Some((lit.node.clone(), lit.token, lit.suffix)),
692-
_ => None,
693-
},
694-
_ => None,
695-
},
696-
Token::Literal(lit, suf) => {
697-
let (suffix_illegal, result) = parse::lit_token(lit, suf, None);
698-
if result.is_none() || suffix_illegal && suf.is_some() {
699-
return None;
680+
impl Lit {
681+
/// Converts literal token with a suffix into an AST literal.
682+
/// Works speculatively and may return `None` is diagnostic handler is not passed.
683+
/// If diagnostic handler is passed, may return `Some`,
684+
/// possibly after reporting non-fatal errors and recovery, or `None` for irrecoverable errors.
685+
crate fn from_token(
686+
token: &token::Token,
687+
span: Span,
688+
diag: Option<(Span, &Handler)>,
689+
) -> Option<Lit> {
690+
let (token, suffix) = match *token {
691+
token::Ident(ident, false) if ident.name == keywords::True.name() ||
692+
ident.name == keywords::False.name() =>
693+
(token::Bool(ident.name), None),
694+
token::Literal(token, suffix) =>
695+
(token, suffix),
696+
token::Interpolated(ref nt) => {
697+
if let token::NtExpr(expr) | token::NtLiteral(expr) = &**nt {
698+
if let ast::ExprKind::Lit(lit) = &expr.node {
699+
return Some(lit.clone());
700+
}
700701
}
701-
Some((result.unwrap(), lit, suf))
702+
return None;
702703
}
703-
_ => None,
704-
}
704+
_ => return None,
705+
};
706+
707+
let node = LitKind::from_lit_token(token, suffix, diag)?;
708+
Some(Lit { node, token, suffix, span })
709+
}
710+
711+
/// Attempts to recover an AST literal from semantic literal.
712+
/// This function is used when the original token doesn't exist (e.g. the literal is created
713+
/// by an AST-based macro) or unavailable (e.g. from HIR pretty-printing).
714+
pub fn from_lit_kind(node: LitKind, span: Span) -> Lit {
715+
let (token, suffix) = node.to_lit_token();
716+
Lit { node, token, suffix, span }
705717
}
706718
}
707719

src/libsyntax/ext/build.rs

Lines changed: 5 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -697,9 +697,9 @@ impl<'a> AstBuilder for ExtCtxt<'a> {
697697
self.expr_struct(span, self.path_ident(span, id), fields)
698698
}
699699

700-
fn expr_lit(&self, span: Span, node: ast::LitKind) -> P<ast::Expr> {
701-
let (token, suffix) = node.lit_token();
702-
self.expr(span, ast::ExprKind::Lit(ast::Lit { node, token, suffix, span }))
700+
fn expr_lit(&self, span: Span, lit_kind: ast::LitKind) -> P<ast::Expr> {
701+
let lit = ast::Lit::from_lit_kind(lit_kind, span);
702+
self.expr(span, ast::ExprKind::Lit(lit))
703703
}
704704
fn expr_usize(&self, span: Span, i: usize) -> P<ast::Expr> {
705705
self.expr_lit(span, ast::LitKind::Int(i as u128,
@@ -1165,11 +1165,10 @@ impl<'a> AstBuilder for ExtCtxt<'a> {
11651165
attr::mk_list_item(sp, Ident::with_empty_ctxt(name).with_span_pos(sp), mis)
11661166
}
11671167

1168-
fn meta_name_value(&self, span: Span, name: ast::Name, node: ast::LitKind)
1168+
fn meta_name_value(&self, span: Span, name: ast::Name, lit_kind: ast::LitKind)
11691169
-> ast::MetaItem {
1170-
let (token, suffix) = node.lit_token();
11711170
attr::mk_name_value_item(span, Ident::with_empty_ctxt(name).with_span_pos(span),
1172-
ast::Lit { node, token, suffix, span })
1171+
lit_kind, span)
11731172
}
11741173

11751174
fn item_use(&self, sp: Span,

0 commit comments

Comments
 (0)