From 9a4d586d1eff5a2abb71e9a12f9ac2515f3bc69d Mon Sep 17 00:00:00 2001 From: Jeong YunWon Date: Sun, 28 May 2023 20:45:21 +0900 Subject: [PATCH] Parse Trait --- parser/README.md | 5 +- parser/src/context.rs | 38 +++--- parser/src/function.rs | 6 +- parser/src/lexer.rs | 2 +- parser/src/lib.rs | 12 +- parser/src/parser.rs | 262 ++++++++++++++++++++++++++++++++++------- parser/src/string.rs | 50 ++++---- 7 files changed, 278 insertions(+), 97 deletions(-) diff --git a/parser/README.md b/parser/README.md index ebe9eb68..e79eb87f 100644 --- a/parser/README.md +++ b/parser/README.md @@ -60,7 +60,8 @@ Execution mode check. Allowed modes are `exec`, `eval` or `single`. For example, one could do this: ``` - use rustpython_parser::{parser, ast}; + use rustpython_parser::{Parse, ast}; let python_source = "print('Hello world')"; - let python_ast = parser::parse_expression(python_source).unwrap(); + let python_statements = ast::Suite::parse(python_source).unwrap(); // statements + let python_expr = ast::Expr::parse(python_source).unwrap(); // or expr ``` diff --git a/parser/src/context.rs b/parser/src/context.rs index 65734d25..90136249 100644 --- a/parser/src/context.rs +++ b/parser/src/context.rs @@ -49,19 +49,19 @@ pub(crate) fn set_context(expr: Expr, ctx: ExprContext) -> Expr { #[cfg(test)] mod tests { - use crate::parser::parse_program; + use crate::{ast, Parse}; #[test] fn test_assign_name() { let source = "x = (1, 2, 3)"; - let parse_ast = parse_program(source, "").unwrap(); + let parse_ast = ast::Suite::parse(source, "").unwrap(); insta::assert_debug_snapshot!(parse_ast); } #[test] fn test_assign_tuple() { let source = "(x, y) = (1, 2, 3)"; - let parse_ast = parse_program(source, "").unwrap(); + let parse_ast = ast::Suite::parse(source, "").unwrap(); insta::assert_debug_snapshot!(parse_ast); } @@ -69,35 +69,35 @@ mod tests { #[cfg(not(feature = "all-nodes-with-ranges"))] fn test_assign_list() { let source = "[x, y] = (1, 2, 3)"; - let parse_ast = parse_program(source, "").unwrap(); + let parse_ast = ast::Suite::parse(source, "").unwrap(); insta::assert_debug_snapshot!(parse_ast); } #[test] fn test_assign_attribute() { let source = "x.y = (1, 2, 3)"; - let parse_ast = parse_program(source, "").unwrap(); + let parse_ast = ast::Suite::parse(source, "").unwrap(); insta::assert_debug_snapshot!(parse_ast); } #[test] fn test_assign_subscript() { let source = "x[y] = (1, 2, 3)"; - let parse_ast = parse_program(source, "").unwrap(); + let parse_ast = ast::Suite::parse(source, "").unwrap(); insta::assert_debug_snapshot!(parse_ast); } #[test] fn test_assign_starred() { let source = "(x, *y) = (1, 2, 3)"; - let parse_ast = parse_program(source, "").unwrap(); + let parse_ast = ast::Suite::parse(source, "").unwrap(); insta::assert_debug_snapshot!(parse_ast); } #[test] fn test_assign_for() { let source = "for x in (1, 2, 3): pass"; - let parse_ast = parse_program(source, "").unwrap(); + let parse_ast = ast::Suite::parse(source, "").unwrap(); insta::assert_debug_snapshot!(parse_ast); } @@ -105,7 +105,7 @@ mod tests { #[cfg(not(feature = "all-nodes-with-ranges"))] fn test_assign_list_comp() { let source = "x = [y for y in (1, 2, 3)]"; - let parse_ast = parse_program(source, "").unwrap(); + let parse_ast = ast::Suite::parse(source, "").unwrap(); insta::assert_debug_snapshot!(parse_ast); } @@ -113,7 +113,7 @@ mod tests { #[cfg(not(feature = "all-nodes-with-ranges"))] fn test_assign_set_comp() { let source = "x = {y for y in (1, 2, 3)}"; - let parse_ast = parse_program(source, "").unwrap(); + let parse_ast = ast::Suite::parse(source, "").unwrap(); insta::assert_debug_snapshot!(parse_ast); } @@ -121,63 +121,63 @@ mod tests { #[cfg(not(feature = "all-nodes-with-ranges"))] fn test_assign_with() { let source = "with 1 as x: pass"; - let parse_ast = parse_program(source, "").unwrap(); + let parse_ast = ast::Suite::parse(source, "").unwrap(); insta::assert_debug_snapshot!(parse_ast); } #[test] fn test_assign_named_expr() { let source = "if x:= 1: pass"; - let parse_ast = parse_program(source, "").unwrap(); + let parse_ast = ast::Suite::parse(source, "").unwrap(); insta::assert_debug_snapshot!(parse_ast); } #[test] fn test_ann_assign_name() { let source = "x: int = 1"; - let parse_ast = parse_program(source, "").unwrap(); + let parse_ast = ast::Suite::parse(source, "").unwrap(); insta::assert_debug_snapshot!(parse_ast); } #[test] fn test_aug_assign_name() { let source = "x += 1"; - let parse_ast = parse_program(source, "").unwrap(); + let parse_ast = ast::Suite::parse(source, "").unwrap(); insta::assert_debug_snapshot!(parse_ast); } #[test] fn test_aug_assign_attribute() { let source = "x.y += (1, 2, 3)"; - let parse_ast = parse_program(source, "").unwrap(); + let parse_ast = ast::Suite::parse(source, "").unwrap(); insta::assert_debug_snapshot!(parse_ast); } #[test] fn test_aug_assign_subscript() { let source = "x[y] += (1, 2, 3)"; - let parse_ast = parse_program(source, "").unwrap(); + let parse_ast = ast::Suite::parse(source, "").unwrap(); insta::assert_debug_snapshot!(parse_ast); } #[test] fn test_del_name() { let source = "del x"; - let parse_ast = parse_program(source, "").unwrap(); + let parse_ast = ast::Suite::parse(source, "").unwrap(); insta::assert_debug_snapshot!(parse_ast); } #[test] fn test_del_attribute() { let source = "del x.y"; - let parse_ast = parse_program(source, "").unwrap(); + let parse_ast = ast::Suite::parse(source, "").unwrap(); insta::assert_debug_snapshot!(parse_ast); } #[test] fn test_del_subscript() { let source = "del x[y]"; - let parse_ast = parse_program(source, "").unwrap(); + let parse_ast = ast::Suite::parse(source, "").unwrap(); insta::assert_debug_snapshot!(parse_ast); } } diff --git a/parser/src/function.rs b/parser/src/function.rs index d3f24f8a..132e7aee 100644 --- a/parser/src/function.rs +++ b/parser/src/function.rs @@ -155,7 +155,7 @@ const fn is_starred(exp: &ast::Expr) -> bool { #[cfg(test)] mod tests { use super::*; - use crate::parser::{parse_program, ParseErrorType}; + use crate::{ast, parser::ParseErrorType, Parse}; #[cfg(not(feature = "all-nodes-with-ranges"))] macro_rules! function_and_lambda { @@ -163,7 +163,7 @@ mod tests { $( #[test] fn $name() { - let parse_ast = parse_program($code, ""); + let parse_ast = ast::Suite::parse($code, ""); insta::assert_debug_snapshot!(parse_ast); } )* @@ -190,7 +190,7 @@ mod tests { } fn function_parse_error(src: &str) -> LexicalErrorType { - let parse_ast = parse_program(src, ""); + let parse_ast = ast::Suite::parse(src, ""); parse_ast .map_err(|e| match e.error { ParseErrorType::Lexical(e) => e, diff --git a/parser/src/lexer.rs b/parser/src/lexer.rs index 1f24ce24..fac0dc05 100644 --- a/parser/src/lexer.rs +++ b/parser/src/lexer.rs @@ -212,7 +212,7 @@ pub fn lex_starts_at( source: &str, mode: Mode, start_offset: TextSize, -) -> impl Iterator + '_ { +) -> SoftKeywordTransformer>> { SoftKeywordTransformer::new(Lexer::new(source.chars(), start_offset), mode) } diff --git a/parser/src/lib.rs b/parser/src/lib.rs index 6268d076..53cec986 100644 --- a/parser/src/lib.rs +++ b/parser/src/lib.rs @@ -94,13 +94,13 @@ //! mode or tokenizing the source beforehand: //! //! ``` -//! use rustpython_parser::parse_program; +//! use rustpython_parser::{Parse, ast}; //! //! let python_source = r#" //! def is_odd(i): //! return bool(i & 1) //! "#; -//! let ast = parse_program(python_source, ""); +//! let ast = ast::Suite::parse(python_source, ""); //! //! assert!(ast.is_ok()); //! ``` @@ -126,13 +126,13 @@ mod soft_keywords; mod string; mod token; -pub use parser::{ - parse, parse_expression, parse_expression_starts_at, parse_program, parse_starts_at, - parse_tokens, ParseError, ParseErrorType, -}; +pub use parser::{parse, parse_starts_at, parse_tokens, Parse, ParseError, ParseErrorType}; pub use string::FStringErrorType; pub use token::{StringKind, Tok}; +#[allow(deprecated)] +pub use parser::{parse_expression, parse_expression_starts_at, parse_program}; + #[rustfmt::skip] mod python { #![allow(clippy::all)] diff --git a/parser/src/parser.rs b/parser/src/parser.rs index 77f836e4..60001e0c 100644 --- a/parser/src/parser.rs +++ b/parser/src/parser.rs @@ -13,7 +13,7 @@ //! [`Mode`]: crate::mode use crate::{ - ast, + ast::{self, OptionalRange, Ranged}, lexer::{self, LexResult, LexicalError, LexicalErrorType}, python, text_size::TextSize, @@ -23,9 +23,179 @@ use crate::{ use itertools::Itertools; use std::iter; -use crate::text_size::TextRange; +use crate::{lexer::Lexer, soft_keywords::SoftKeywordTransformer, text_size::TextRange}; pub(super) use lalrpop_util::ParseError as LalrpopError; -use rustpython_ast::OptionalRange; + +/// Parse Python code string to implementor's type. +/// +/// # Example +/// +/// For example, parsing a simple function definition and a call to that function: +/// +/// ``` +/// use rustpython_parser::{self as parser, ast, Parse}; +/// let source = r#" +/// def foo(): +/// return 42 +/// +/// print(foo()) +/// "#; +/// let program = ast::Suite::parse(source, ""); +/// assert!(program.is_ok()); +/// ``` +/// +/// Parsing a single expression denoting the addition of two numbers, but this time specifying a different, +/// somewhat silly, location: +/// +/// ``` +/// use rustpython_parser::{self as parser, ast, Parse, text_size::TextSize}; +/// +/// let expr = ast::Expr::parse_starts_at("1 + 2", "", TextSize::from(400)); +/// assert!(expr.is_ok()); +pub trait Parse +where + Self: Sized, +{ + fn parse(source: &str, source_path: &str) -> Result { + Self::parse_starts_at(source, source_path, TextSize::default()) + } + fn parse_starts_at( + source: &str, + source_path: &str, + offset: TextSize, + ) -> Result { + let lxr = Self::lex_starts_at(source, offset); + #[cfg(feature = "full-lexer")] + let lxr = + lxr.filter_ok(|(tok, _)| !matches!(tok, Tok::Comment { .. } | Tok::NonLogicalNewline)); + Self::parse_tokens(lxr, source_path) + } + fn lex_starts_at( + source: &str, + offset: TextSize, + ) -> SoftKeywordTransformer>; + fn parse_tokens( + lxr: impl IntoIterator, + source_path: &str, + ) -> Result; +} + +impl Parse for ast::ModModule { + fn lex_starts_at( + source: &str, + offset: TextSize, + ) -> SoftKeywordTransformer> { + lexer::lex_starts_at(source, Mode::Module, offset) + } + fn parse_tokens( + lxr: impl IntoIterator, + source_path: &str, + ) -> Result { + match parse_filtered_tokens(lxr, Mode::Module, source_path)? { + ast::Mod::Module(m) => Ok(m), + _ => unreachable!("Mode::Module doesn't return other variant"), + } + } +} + +impl Parse for ast::ModExpression { + fn lex_starts_at( + source: &str, + offset: TextSize, + ) -> SoftKeywordTransformer> { + lexer::lex_starts_at(source, Mode::Expression, offset) + } + fn parse_tokens( + lxr: impl IntoIterator, + source_path: &str, + ) -> Result { + match parse_filtered_tokens(lxr, Mode::Expression, source_path)? { + ast::Mod::Expression(m) => Ok(m), + _ => unreachable!("Mode::Module doesn't return other variant"), + } + } +} + +impl Parse for ast::ModInteractive { + fn lex_starts_at( + source: &str, + offset: TextSize, + ) -> SoftKeywordTransformer> { + lexer::lex_starts_at(source, Mode::Interactive, offset) + } + fn parse_tokens( + lxr: impl IntoIterator, + source_path: &str, + ) -> Result { + match parse_filtered_tokens(lxr, Mode::Interactive, source_path)? { + ast::Mod::Interactive(m) => Ok(m), + _ => unreachable!("Mode::Module doesn't return other variant"), + } + } +} + +impl Parse for ast::Suite { + fn lex_starts_at( + source: &str, + offset: TextSize, + ) -> SoftKeywordTransformer> { + ast::ModModule::lex_starts_at(source, offset) + } + fn parse_tokens( + lxr: impl IntoIterator, + source_path: &str, + ) -> Result { + Ok(ast::ModModule::parse_tokens(lxr, source_path)?.body) + } +} + +impl Parse for ast::Stmt { + fn lex_starts_at( + source: &str, + offset: TextSize, + ) -> SoftKeywordTransformer> { + ast::ModModule::lex_starts_at(source, offset) + } + fn parse_tokens( + lxr: impl IntoIterator, + source_path: &str, + ) -> Result { + let mut statements = ast::ModModule::parse_tokens(lxr, source_path)?.body; + let statement = match statements.len() { + 0 => { + return Err(ParseError { + error: ParseErrorType::Eof, + offset: TextSize::default(), + source_path: source_path.to_owned(), + }) + } + 1 => statements.pop().unwrap(), + _ => { + return Err(ParseError { + error: ParseErrorType::InvalidToken, + offset: statements[1].range().start(), + source_path: source_path.to_owned(), + }) + } + }; + Ok(statement) + } +} + +impl Parse for ast::Expr { + fn lex_starts_at( + source: &str, + offset: TextSize, + ) -> SoftKeywordTransformer> { + ast::ModExpression::lex_starts_at(source, offset) + } + fn parse_tokens( + lxr: impl IntoIterator, + source_path: &str, + ) -> Result { + Ok(*ast::ModExpression::parse_tokens(lxr, source_path)?.body) + } +} /// Parse a full Python program usually consisting of multiple lines. /// @@ -47,6 +217,7 @@ use rustpython_ast::OptionalRange; /// let program = parser::parse_program(source, ""); /// assert!(program.is_ok()); /// ``` +#[deprecated = "Use ast::Suite::parse from rustpython_parser::Parse trait."] pub fn parse_program(source: &str, source_path: &str) -> Result { parse(source, Mode::Module, source_path).map(|top| match top { ast::Mod::Module(ast::ModModule { body, .. }) => body, @@ -71,8 +242,9 @@ pub fn parse_program(source: &str, source_path: &str) -> Result Result { - parse_expression_starts_at(source, path, TextSize::default()) + ast::Expr::parse(source, path) } /// Parses a Python expression from a given location. @@ -91,15 +263,13 @@ pub fn parse_expression(source: &str, path: &str) -> Result", TextSize::from(400)); /// assert!(expr.is_ok()); /// ``` +#[deprecated = "Use ast::Expr::parse_starts_at from rustpython_parser::Parse trait."] pub fn parse_expression_starts_at( source: &str, path: &str, offset: TextSize, ) -> Result { - parse_starts_at(source, Mode::Expression, path, offset).map(|top| match top { - ast::Mod::Expression(ast::ModExpression { body, .. }) => *body, - _ => unreachable!(), - }) + ast::Expr::parse_starts_at(source, path, offset) } /// Parse the given Python source code using the specified [`Mode`]. @@ -188,12 +358,21 @@ pub fn parse_tokens( lxr: impl IntoIterator, mode: Mode, source_path: &str, +) -> Result { + let lxr = lxr.into_iter(); + #[cfg(feature = "full-lexer")] + let lxr = + lxr.filter_ok(|(tok, _)| !matches!(tok, Tok::Comment { .. } | Tok::NonLogicalNewline)); + parse_filtered_tokens(lxr, mode, source_path) +} + +fn parse_filtered_tokens( + lxr: impl IntoIterator, + mode: Mode, + source_path: &str, ) -> Result { let marker_token = (Tok::start_marker(mode), Default::default()); let lexer = iter::once(Ok(marker_token)).chain(lxr); - #[cfg(feature = "full-lexer")] - let lexer = - lexer.filter_ok(|(tok, _)| !matches!(tok, Tok::Comment { .. } | Tok::NonLogicalNewline)); python::TopParser::new() .parse( lexer @@ -329,52 +508,53 @@ pub(super) fn optional_range(start: TextSize, end: TextSize) -> OptionalRange").unwrap(); + let parse_ast = ast::Suite::parse("", "").unwrap(); insta::assert_debug_snapshot!(parse_ast); } #[test] fn test_parse_string() { let source = "'Hello world'"; - let parse_ast = parse_program(source, "").unwrap(); + let parse_ast = ast::Suite::parse(source, "").unwrap(); insta::assert_debug_snapshot!(parse_ast); } #[test] fn test_parse_f_string() { let source = "f'Hello world'"; - let parse_ast = parse_program(source, "").unwrap(); + let parse_ast = ast::Suite::parse(source, "").unwrap(); insta::assert_debug_snapshot!(parse_ast); } #[test] fn test_parse_print_hello() { let source = "print('Hello world')"; - let parse_ast = parse_program(source, "").unwrap(); + let parse_ast = ast::Suite::parse(source, "").unwrap(); insta::assert_debug_snapshot!(parse_ast); } #[test] fn test_parse_print_2() { let source = "print('Hello world', 2)"; - let parse_ast = parse_program(source, "").unwrap(); + let parse_ast = ast::Suite::parse(source, "").unwrap(); insta::assert_debug_snapshot!(parse_ast); } #[test] fn test_parse_kwargs() { let source = "my_func('positional', keyword=2)"; - let parse_ast = parse_program(source, "").unwrap(); + let parse_ast = ast::Suite::parse(source, "").unwrap(); insta::assert_debug_snapshot!(parse_ast); } #[test] fn test_parse_if_elif_else() { let source = "if 1: 10\nelif 2: 20\nelse: 30"; - let parse_ast = parse_program(source, "").unwrap(); + let parse_ast = ast::Suite::parse(source, "").unwrap(); insta::assert_debug_snapshot!(parse_ast); } @@ -382,7 +562,7 @@ mod tests { #[cfg(not(feature = "all-nodes-with-ranges"))] fn test_parse_lambda() { let source = "lambda x, y: x * y"; // lambda(x, y): x * y"; - let parse_ast = parse_program(source, "").unwrap(); + let parse_ast = ast::Suite::parse(source, "").unwrap(); insta::assert_debug_snapshot!(parse_ast); } @@ -390,7 +570,7 @@ mod tests { fn test_parse_tuples() { let source = "a, b = 4, 5"; - insta::assert_debug_snapshot!(parse_program(source, "").unwrap()); + insta::assert_debug_snapshot!(ast::Suite::parse(source, "").unwrap()); } #[test] @@ -403,14 +583,14 @@ class Foo(A, B): def method_with_default(self, arg='default'): pass "; - insta::assert_debug_snapshot!(parse_program(source, "").unwrap()); + insta::assert_debug_snapshot!(ast::Suite::parse(source, "").unwrap()); } #[test] #[cfg(not(feature = "all-nodes-with-ranges"))] fn test_parse_dict_comprehension() { let source = "{x1: x2 for y in z}"; - let parse_ast = parse_expression(source, "").unwrap(); + let parse_ast = ast::Expr::parse(source, "").unwrap(); insta::assert_debug_snapshot!(parse_ast); } @@ -418,7 +598,7 @@ class Foo(A, B): #[cfg(not(feature = "all-nodes-with-ranges"))] fn test_parse_list_comprehension() { let source = "[x for y in z]"; - let parse_ast = parse_expression(source, "").unwrap(); + let parse_ast = ast::Expr::parse(source, "").unwrap(); insta::assert_debug_snapshot!(parse_ast); } @@ -426,7 +606,7 @@ class Foo(A, B): #[cfg(not(feature = "all-nodes-with-ranges"))] fn test_parse_double_list_comprehension() { let source = "[x for y, y2 in z for a in b if a < 5 if a > 10]"; - let parse_ast = parse_expression(source, "").unwrap(); + let parse_ast = ast::Expr::parse(source, "").unwrap(); insta::assert_debug_snapshot!(parse_ast); } @@ -434,7 +614,7 @@ class Foo(A, B): #[cfg(not(feature = "all-nodes-with-ranges"))] fn test_parse_generator_comprehension() { let source = "(x for y in z)"; - let parse_ast = parse_expression(source, "").unwrap(); + let parse_ast = ast::Expr::parse(source, "").unwrap(); insta::assert_debug_snapshot!(parse_ast); } @@ -442,7 +622,7 @@ class Foo(A, B): #[cfg(not(feature = "all-nodes-with-ranges"))] fn test_parse_named_expression_generator_comprehension() { let source = "(x := y + 1 for y in z)"; - let parse_ast = parse_expression(source, "").unwrap(); + let parse_ast = ast::Expr::parse(source, "").unwrap(); insta::assert_debug_snapshot!(parse_ast); } @@ -450,28 +630,28 @@ class Foo(A, B): #[cfg(not(feature = "all-nodes-with-ranges"))] fn test_parse_if_else_generator_comprehension() { let source = "(x if y else y for y in z)"; - let parse_ast = parse_expression(source, "").unwrap(); + let parse_ast = ast::Expr::parse(source, "").unwrap(); insta::assert_debug_snapshot!(parse_ast); } #[test] fn test_parse_bool_op_or() { let source = "x or y"; - let parse_ast = parse_expression(source, "").unwrap(); + let parse_ast = ast::Expr::parse(source, "").unwrap(); insta::assert_debug_snapshot!(parse_ast); } #[test] fn test_parse_bool_op_and() { let source = "x and y"; - let parse_ast = parse_expression(source, "").unwrap(); + let parse_ast = ast::Expr::parse(source, "").unwrap(); insta::assert_debug_snapshot!(parse_ast); } #[test] fn test_slice() { let source = "x[1:2:3]"; - let parse_ast = parse_expression(source, "").unwrap(); + let parse_ast = ast::Expr::parse(source, "").unwrap(); insta::assert_debug_snapshot!(parse_ast); } @@ -506,7 +686,7 @@ with (0 as a,): pass with (0 as a, 1 as b): pass with (0 as a, 1 as b,): pass "; - insta::assert_debug_snapshot!(parse_program(source, "").unwrap()); + insta::assert_debug_snapshot!(ast::Suite::parse(source, "").unwrap()); } #[test] @@ -529,7 +709,7 @@ with (0 as a, 1 as b,): pass "with a := 0 as x: pass", "with (a := 0 as x): pass", ] { - assert!(parse_program(source, "").is_err()); + assert!(ast::Suite::parse(source, "").is_err()); } } @@ -541,7 +721,7 @@ array[0, *indexes, -1] = array_slice array[*indexes_to_select, *indexes_to_select] array[3:5, *indexes_to_select] "; - let parse_ast = parse_program(source, "").unwrap(); + let parse_ast = ast::Suite::parse(source, "").unwrap(); insta::assert_debug_snapshot!(parse_ast); } @@ -555,13 +735,13 @@ array[3:5, *indexes_to_select] ("OFFSET %d" % offset) if offset else None, ) )"#; - let parse_ast = parse_expression(source, "").unwrap(); + let parse_ast = ast::Expr::parse(source, "").unwrap(); insta::assert_debug_snapshot!(parse_ast); } #[test] fn test_try() { - let parse_ast = parse_program( + let parse_ast = ast::Suite::parse( r#"try: raise ValueError(1) except TypeError as e: @@ -576,7 +756,7 @@ except OSError as e: #[test] fn test_try_star() { - let parse_ast = parse_program( + let parse_ast = ast::Suite::parse( r#"try: raise ExceptionGroup("eg", [ValueError(1), TypeError(2), OSError(3), OSError(4)]) @@ -592,7 +772,7 @@ except* OSError as e: #[test] fn test_dict_unpacking() { - let parse_ast = parse_expression(r#"{"a": "b", **c, "d": "e"}"#, "").unwrap(); + let parse_ast = ast::Expr::parse(r#"{"a": "b", **c, "d": "e"}"#, "").unwrap(); insta::assert_debug_snapshot!(parse_ast); } @@ -608,7 +788,7 @@ except* OSError as e: #[test] #[cfg(not(feature = "all-nodes-with-ranges"))] fn test_match_as_identifier() { - let parse_ast = parse_program( + let parse_ast = ast::Suite::parse( r#" match *a + b, c # ((match * a) + b), c match *(a + b), c # (match * (a + b)), c @@ -806,14 +986,14 @@ match w := x,: case y as v,: z = 0 "#; - let parse_ast = parse_program(source, "").unwrap(); + let parse_ast = ast::Suite::parse(source, "").unwrap(); insta::assert_debug_snapshot!(parse_ast); } #[test] #[cfg(not(feature = "all-nodes-with-ranges"))] fn test_match() { - let parse_ast = parse_program( + let parse_ast = ast::Suite::parse( r#" match {"test": 1}: case { @@ -844,7 +1024,7 @@ match x: #[test] #[cfg(not(feature = "all-nodes-with-ranges"))] fn test_variadic_generics() { - let parse_ast = parse_program( + let parse_ast = ast::Suite::parse( r#" def args_to_tuple(*args: *Ts) -> Tuple[*Ts]: ... "#, diff --git a/parser/src/string.rs b/parser/src/string.rs index f575dc37..cb2b0e19 100644 --- a/parser/src/string.rs +++ b/parser/src/string.rs @@ -7,7 +7,7 @@ use crate::text_size::TextRange; use crate::{ ast::{self, Constant, Expr}, lexer::{LexicalError, LexicalErrorType}, - parser::{parse_expression_starts_at, LalrpopError, ParseError, ParseErrorType}, + parser::{LalrpopError, Parse, ParseError, ParseErrorType}, token::{StringKind, Tok}, }; use itertools::Itertools; @@ -591,7 +591,7 @@ impl<'a> StringParser<'a> { fn parse_fstring_expr(source: &str, location: TextSize) -> Result { let fstring_body = format!("({source})"); let start = location - TextSize::from(1); - parse_expression_starts_at(&fstring_body, "", start) + ast::Expr::parse_starts_at(&fstring_body, "", start) } fn parse_string( @@ -815,7 +815,7 @@ impl From for LalrpopError { #[cfg(test)] mod tests { use super::*; - use crate::parser::parse_program; + use crate::{ast, Parse}; fn parse_fstring(source: &str) -> Result, LexicalError> { StringParser::new( @@ -957,63 +957,63 @@ mod tests { #[test] fn test_parse_string_concat() { let source = "'Hello ' 'world'"; - let parse_ast = parse_program(source, "").unwrap(); + let parse_ast = ast::Suite::parse(source, "").unwrap(); insta::assert_debug_snapshot!(parse_ast); } #[test] fn test_parse_u_string_concat_1() { let source = "'Hello ' u'world'"; - let parse_ast = parse_program(source, "").unwrap(); + let parse_ast = ast::Suite::parse(source, "").unwrap(); insta::assert_debug_snapshot!(parse_ast); } #[test] fn test_parse_u_string_concat_2() { let source = "u'Hello ' 'world'"; - let parse_ast = parse_program(source, "").unwrap(); + let parse_ast = ast::Suite::parse(source, "").unwrap(); insta::assert_debug_snapshot!(parse_ast); } #[test] fn test_parse_f_string_concat_1() { let source = "'Hello ' f'world'"; - let parse_ast = parse_program(source, "").unwrap(); + let parse_ast = ast::Suite::parse(source, "").unwrap(); insta::assert_debug_snapshot!(parse_ast); } #[test] fn test_parse_f_string_concat_2() { let source = "'Hello ' f'world'"; - let parse_ast = parse_program(source, "").unwrap(); + let parse_ast = ast::Suite::parse(source, "").unwrap(); insta::assert_debug_snapshot!(parse_ast); } #[test] fn test_parse_f_string_concat_3() { let source = "'Hello ' f'world{\"!\"}'"; - let parse_ast = parse_program(source, "").unwrap(); + let parse_ast = ast::Suite::parse(source, "").unwrap(); insta::assert_debug_snapshot!(parse_ast); } #[test] fn test_parse_u_f_string_concat_1() { let source = "u'Hello ' f'world'"; - let parse_ast = parse_program(source, "").unwrap(); + let parse_ast = ast::Suite::parse(source, "").unwrap(); insta::assert_debug_snapshot!(parse_ast); } #[test] fn test_parse_u_f_string_concat_2() { let source = "u'Hello ' f'world' '!'"; - let parse_ast = parse_program(source, "").unwrap(); + let parse_ast = ast::Suite::parse(source, "").unwrap(); insta::assert_debug_snapshot!(parse_ast); } #[test] fn test_parse_string_triple_quotes_with_kind() { let source = "u'''Hello, world!'''"; - let parse_ast = parse_program(source, "").unwrap(); + let parse_ast = ast::Suite::parse(source, "").unwrap(); insta::assert_debug_snapshot!(parse_ast); } @@ -1021,7 +1021,7 @@ mod tests { fn test_single_quoted_byte() { // single quote let source = r##"b'\x00\x01\x02\x03\x04\x05\x06\x07\x08\t\n\x0b\x0c\r\x0e\x0f\x10\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f !"#$%&\'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\\]^_`abcdefghijklmnopqrstuvwxyz{|}~\x7f\x80\x81\x82\x83\x84\x85\x86\x87\x88\x89\x8a\x8b\x8c\x8d\x8e\x8f\x90\x91\x92\x93\x94\x95\x96\x97\x98\x99\x9a\x9b\x9c\x9d\x9e\x9f\xa0\xa1\xa2\xa3\xa4\xa5\xa6\xa7\xa8\xa9\xaa\xab\xac\xad\xae\xaf\xb0\xb1\xb2\xb3\xb4\xb5\xb6\xb7\xb8\xb9\xba\xbb\xbc\xbd\xbe\xbf\xc0\xc1\xc2\xc3\xc4\xc5\xc6\xc7\xc8\xc9\xca\xcb\xcc\xcd\xce\xcf\xd0\xd1\xd2\xd3\xd4\xd5\xd6\xd7\xd8\xd9\xda\xdb\xdc\xdd\xde\xdf\xe0\xe1\xe2\xe3\xe4\xe5\xe6\xe7\xe8\xe9\xea\xeb\xec\xed\xee\xef\xf0\xf1\xf2\xf3\xf4\xf5\xf6\xf7\xf8\xf9\xfa\xfb\xfc\xfd\xfe\xff'"##; - let parse_ast = parse_program(source, "").unwrap(); + let parse_ast = ast::Suite::parse(source, "").unwrap(); insta::assert_debug_snapshot!(parse_ast); } @@ -1029,7 +1029,7 @@ mod tests { fn test_double_quoted_byte() { // double quote let source = r##"b"\x00\x01\x02\x03\x04\x05\x06\x07\x08\t\n\x0b\x0c\r\x0e\x0f\x10\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f !\"#$%&'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\\]^_`abcdefghijklmnopqrstuvwxyz{|}~\x7f\x80\x81\x82\x83\x84\x85\x86\x87\x88\x89\x8a\x8b\x8c\x8d\x8e\x8f\x90\x91\x92\x93\x94\x95\x96\x97\x98\x99\x9a\x9b\x9c\x9d\x9e\x9f\xa0\xa1\xa2\xa3\xa4\xa5\xa6\xa7\xa8\xa9\xaa\xab\xac\xad\xae\xaf\xb0\xb1\xb2\xb3\xb4\xb5\xb6\xb7\xb8\xb9\xba\xbb\xbc\xbd\xbe\xbf\xc0\xc1\xc2\xc3\xc4\xc5\xc6\xc7\xc8\xc9\xca\xcb\xcc\xcd\xce\xcf\xd0\xd1\xd2\xd3\xd4\xd5\xd6\xd7\xd8\xd9\xda\xdb\xdc\xdd\xde\xdf\xe0\xe1\xe2\xe3\xe4\xe5\xe6\xe7\xe8\xe9\xea\xeb\xec\xed\xee\xef\xf0\xf1\xf2\xf3\xf4\xf5\xf6\xf7\xf8\xf9\xfa\xfb\xfc\xfd\xfe\xff""##; - let parse_ast = parse_program(source, "").unwrap(); + let parse_ast = ast::Suite::parse(source, "").unwrap(); insta::assert_debug_snapshot!(parse_ast); } @@ -1037,35 +1037,35 @@ mod tests { fn test_escape_char_in_byte_literal() { // backslash does not escape let source = r##"b"omkmok\Xaa""##; // spell-checker:ignore omkmok - let parse_ast = parse_program(source, "").unwrap(); + let parse_ast = ast::Suite::parse(source, "").unwrap(); insta::assert_debug_snapshot!(parse_ast); } #[test] fn test_raw_byte_literal_1() { let source = r"rb'\x1z'"; - let parse_ast = parse_program(source, "").unwrap(); + let parse_ast = ast::Suite::parse(source, "").unwrap(); insta::assert_debug_snapshot!(parse_ast); } #[test] fn test_raw_byte_literal_2() { let source = r"rb'\\'"; - let parse_ast = parse_program(source, "").unwrap(); + let parse_ast = ast::Suite::parse(source, "").unwrap(); insta::assert_debug_snapshot!(parse_ast); } #[test] fn test_escape_octet() { let source = r##"b'\43a\4\1234'"##; - let parse_ast = parse_program(source, "").unwrap(); + let parse_ast = ast::Suite::parse(source, "").unwrap(); insta::assert_debug_snapshot!(parse_ast); } #[test] fn test_fstring_escaped_newline() { let source = r#"f"\n{x}""#; - let parse_ast = parse_program(source, "").unwrap(); + let parse_ast = ast::Suite::parse(source, "").unwrap(); insta::assert_debug_snapshot!(parse_ast); } @@ -1073,28 +1073,28 @@ mod tests { fn test_fstring_unescaped_newline() { let source = r#"f""" {x}""""#; - let parse_ast = parse_program(source, "").unwrap(); + let parse_ast = ast::Suite::parse(source, "").unwrap(); insta::assert_debug_snapshot!(parse_ast); } #[test] fn test_fstring_escaped_character() { let source = r#"f"\\{x}""#; - let parse_ast = parse_program(source, "").unwrap(); + let parse_ast = ast::Suite::parse(source, "").unwrap(); insta::assert_debug_snapshot!(parse_ast); } #[test] fn test_raw_fstring() { let source = r#"rf"{x}""#; - let parse_ast = parse_program(source, "").unwrap(); + let parse_ast = ast::Suite::parse(source, "").unwrap(); insta::assert_debug_snapshot!(parse_ast); } #[test] fn test_triple_quoted_raw_fstring() { let source = r#"rf"""{x}""""#; - let parse_ast = parse_program(source, "").unwrap(); + let parse_ast = ast::Suite::parse(source, "").unwrap(); insta::assert_debug_snapshot!(parse_ast); } @@ -1102,7 +1102,7 @@ mod tests { fn test_fstring_line_continuation() { let source = r#"rf"\ {x}""#; - let parse_ast = parse_program(source, "").unwrap(); + let parse_ast = ast::Suite::parse(source, "").unwrap(); insta::assert_debug_snapshot!(parse_ast); } @@ -1112,7 +1112,7 @@ mod tests { #[test] fn $name() { let source = format!(r#""\N{{{0}}}""#, $alias); - let parse_ast = parse_program(&source, "").unwrap(); + let parse_ast = ast::Suite::parse(&source, "").unwrap(); insta::assert_debug_snapshot!(parse_ast); } )*