From 3b1077a445181d6c7f934db517c3bcb20636b88f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ond=C5=99ej=20=C4=8Cert=C3=ADk?= Date: Fri, 23 Sep 2022 13:43:17 -0400 Subject: [PATCH 1/8] Switch to the new parser --- src/lpython/parser/parser.cpp | 35 ++++++++++------------------------- 1 file changed, 10 insertions(+), 25 deletions(-) diff --git a/src/lpython/parser/parser.cpp b/src/lpython/parser/parser.cpp index f5bbf0408e..a5991564d6 100644 --- a/src/lpython/parser/parser.cpp +++ b/src/lpython/parser/parser.cpp @@ -113,36 +113,21 @@ std::string unique_filename(const std::string &prefix) { } Result parse_python_file(Allocator &al, - const std::string &runtime_library_dir, + const std::string &/*runtime_library_dir*/, const std::string &infile, diag::Diagnostics &diagnostics, bool new_parser) { LPython::AST::ast_t* ast; - if (new_parser) { - std::string input = read_file(infile); - Result res = parse(al, input, diagnostics); - if (res.ok) { - ast = (LPython::AST::ast_t*)res.result; - } else { - LFORTRAN_ASSERT(diagnostics.has_error()) - return Error(); - } + // We will be using the new parser from now on + new_parser = true; + LFORTRAN_ASSERT(new_parser) + std::string input = read_file(infile); + Result res = parse(al, input, diagnostics); + if (res.ok) { + ast = (LPython::AST::ast_t*)res.result; } else { - std::string outfile = unique_filename(infile); - std::string pycmd = "python " + runtime_library_dir - + "/lpython_parser.py " + infile + " " + outfile; - int err = std::system(pycmd.c_str()); - if (err != 0) { - std::cerr << "The command '" << pycmd << "' failed." << std::endl; - return Error(); - } - std::string input; - bool status = read_file(outfile, input); - if (!status) { - std::cerr << "The file '" << outfile << "' cannot be read." << std::endl; - return Error(); - } - ast = LPython::deserialize_ast(al, input); + LFORTRAN_ASSERT(diagnostics.has_error()) + return Error(); } return ast; } From 6f03ce0ca9b8c126d98b76ec3b6d4332d99e3271 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ond=C5=99ej=20=C4=8Cert=C3=ADk?= Date: Fri, 23 Sep 2022 13:49:08 -0400 Subject: [PATCH 2/8] Use LLOC to skip dedent --- src/lpython/parser/parser.yy | 2 +- src/lpython/parser/semantics.h | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/src/lpython/parser/parser.yy b/src/lpython/parser/parser.yy index 99c9f3bd86..bc9c9347db 100644 --- a/src/lpython/parser/parser.yy +++ b/src/lpython/parser/parser.yy @@ -315,7 +315,7 @@ script_unit ; statements - : TK_INDENT statements1 TK_DEDENT { $$ = $2; } + : TK_INDENT statements1 TK_DEDENT { $$ = $2; LLOC(@$,$2); } ; sep_statements diff --git a/src/lpython/parser/semantics.h b/src/lpython/parser/semantics.h index 5311b930f9..84fe98492e 100644 --- a/src/lpython/parser/semantics.h +++ b/src/lpython/parser/semantics.h @@ -58,6 +58,10 @@ static inline T** vec_cast(const Vec &x) { return s; } +// Assign last location to `a` from `b` +#define LLOC(a, b) a.last = b.last; + + #define VEC_CAST(x, type) vec_cast(x) #define STMTS(x) VEC_CAST(x, stmt) #define EXPRS(x) VEC_CAST(x, expr) From e7a44de84e7896258b65492fd201bd919a7e3276 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ond=C5=99ej=20=C4=8Cert=C3=ADk?= Date: Fri, 23 Sep 2022 14:00:08 -0400 Subject: [PATCH 3/8] Fix location information --- src/lpython/parser/parser.yy | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/src/lpython/parser/parser.yy b/src/lpython/parser/parser.yy index bc9c9347db..8ea7c9777c 100644 --- a/src/lpython/parser/parser.yy +++ b/src/lpython/parser/parser.yy @@ -315,7 +315,8 @@ script_unit ; statements - : TK_INDENT statements1 TK_DEDENT { $$ = $2; LLOC(@$,$2); } + : TK_INDENT statements1 TK_DEDENT { $$ = $2; LLOC(@$,@2); + @$.first = @2.first; } ; sep_statements @@ -334,9 +335,9 @@ statements1 ; single_line_statements - : single_line_multi_statements TK_NEWLINE { $$ = $1; } + : single_line_multi_statements TK_NEWLINE { $$ = $1; LLOC(@$,@1); } | single_line_multi_statements TK_EOLCOMMENT { $$ = $1; } - | single_line_statement TK_NEWLINE { $$ = A2LIST(p.m_a, $1); } + | single_line_statement TK_NEWLINE { $$ = A2LIST(p.m_a, $1); LLOC(@$,@1); } | single_line_statement TK_SEMICOLON TK_NEWLINE { $$ = A2LIST(p.m_a, $1); } | single_line_statement TK_SEMICOLON TK_EOLCOMMENT { $$ = A2LIST(p.m_a, $1); } | single_line_statement TK_EOLCOMMENT { $$ = A2LIST(p.m_a, $1); } @@ -358,10 +359,10 @@ type_ignore_sep ; statement - : single_line_statement sep { $$ = $1; } + : single_line_statement sep { $$ = $1; LLOC(@$,@1); } | single_line_statement type_ignore_sep { $$ = $1; } | multi_line_statement - | multi_line_statement sep { $$ = $1; } + | multi_line_statement sep { $$ = $1; LLOC(@$,@1); } | multi_line_statement type_ignore_sep { $$ = $1; } ; From 60d8ccadcccb2a1271a177d02d917b2bea4922c0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ond=C5=99ej=20=C4=8Cert=C3=ADk?= Date: Fri, 23 Sep 2022 14:08:19 -0400 Subject: [PATCH 4/8] Change the first location information --- src/lpython/parser/parser.yy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lpython/parser/parser.yy b/src/lpython/parser/parser.yy index 8ea7c9777c..78ce5fe2bd 100644 --- a/src/lpython/parser/parser.yy +++ b/src/lpython/parser/parser.yy @@ -720,7 +720,7 @@ class_def : decorators_opt KW_CLASS id ":" body_stmts { $$ = CLASS_01($1, $3, $5, @$); } | decorators_opt KW_CLASS id "(" call_arguement_list ")" ":" body_stmts { - $$ = CLASS_02($1, $3, $5, $8, @$); } + @$.first = @2.first; $$ = CLASS_02($1, $3, $5, $8, @$); } ; async_func_def From dbeef4ec3315bab188695736d97c3724c7bccadb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ond=C5=99ej=20=C4=8Cert=C3=ADk?= Date: Fri, 23 Sep 2022 14:27:27 -0400 Subject: [PATCH 5/8] Move the LLOC at the beginning --- src/lpython/parser/parser.yy | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/src/lpython/parser/parser.yy b/src/lpython/parser/parser.yy index 78ce5fe2bd..50fa7dc905 100644 --- a/src/lpython/parser/parser.yy +++ b/src/lpython/parser/parser.yy @@ -315,13 +315,12 @@ script_unit ; statements - : TK_INDENT statements1 TK_DEDENT { $$ = $2; LLOC(@$,@2); - @$.first = @2.first; } + : TK_INDENT statements1 TK_DEDENT { @$.first = @2.first; LLOC(@$,@2); $$ = $2; } ; sep_statements - : sep statements { $$ = $2; } - | type_ignore_sep statements { $$ = $2; } + : sep statements { @$.first = @2.first; $$ = $2; } + | type_ignore_sep statements { @$.first = @2.first; $$ = $2; } ; body_stmts @@ -335,9 +334,9 @@ statements1 ; single_line_statements - : single_line_multi_statements TK_NEWLINE { $$ = $1; LLOC(@$,@1); } + : single_line_multi_statements TK_NEWLINE { LLOC(@$,@1); $$ = $1; } | single_line_multi_statements TK_EOLCOMMENT { $$ = $1; } - | single_line_statement TK_NEWLINE { $$ = A2LIST(p.m_a, $1); LLOC(@$,@1); } + | single_line_statement TK_NEWLINE { LLOC(@$,@1); $$ = A2LIST(p.m_a, $1); } | single_line_statement TK_SEMICOLON TK_NEWLINE { $$ = A2LIST(p.m_a, $1); } | single_line_statement TK_SEMICOLON TK_EOLCOMMENT { $$ = A2LIST(p.m_a, $1); } | single_line_statement TK_EOLCOMMENT { $$ = A2LIST(p.m_a, $1); } @@ -359,10 +358,10 @@ type_ignore_sep ; statement - : single_line_statement sep { $$ = $1; LLOC(@$,@1); } + : single_line_statement sep { LLOC(@$,@1); $$ = $1; } | single_line_statement type_ignore_sep { $$ = $1; } | multi_line_statement - | multi_line_statement sep { $$ = $1; LLOC(@$,@1); } + | multi_line_statement sep { LLOC(@$,@1); $$ = $1; } | multi_line_statement type_ignore_sep { $$ = $1; } ; From 642da94d52b9bc68e2b7bed7a1dcc786555170b3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ond=C5=99ej=20=C4=8Cert=C3=ADk?= Date: Fri, 23 Sep 2022 14:38:02 -0400 Subject: [PATCH 6/8] Exclude decorators for functions too --- src/lpython/parser/parser.yy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lpython/parser/parser.yy b/src/lpython/parser/parser.yy index 50fa7dc905..2825209051 100644 --- a/src/lpython/parser/parser.yy +++ b/src/lpython/parser/parser.yy @@ -700,7 +700,7 @@ function_def : decorators_opt KW_DEF id "(" parameter_list_opt ")" ":" body_stmts { $$ = FUNCTION_01($1, $3, $5, $8, @$); } | decorators_opt KW_DEF id "(" parameter_list_opt ")" "->" expr ":" - body_stmts { $$ = FUNCTION_02($1, $3, $5, $8, $10, @$); } + body_stmts { @$.first = @2.first; $$ = FUNCTION_02($1, $3, $5, $8, $10, @$); } | decorators_opt KW_DEF id "(" parameter_list_opt ")" ":" TK_TYPE_COMMENT sep statements { $$ = FUNCTION_03($1, $3, $5, $10, $8, @$); } From 76fce89d56cd7856fd1b734538cc8bbefee337d6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ond=C5=99ej=20=C4=8Cert=C3=ADk?= Date: Fri, 23 Sep 2022 14:41:26 -0400 Subject: [PATCH 7/8] Introduce FLOC and use it --- src/lpython/parser/parser.yy | 10 +++++----- src/lpython/parser/semantics.h | 3 ++- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/src/lpython/parser/parser.yy b/src/lpython/parser/parser.yy index 2825209051..c5fe2b7f50 100644 --- a/src/lpython/parser/parser.yy +++ b/src/lpython/parser/parser.yy @@ -315,12 +315,12 @@ script_unit ; statements - : TK_INDENT statements1 TK_DEDENT { @$.first = @2.first; LLOC(@$,@2); $$ = $2; } + : TK_INDENT statements1 TK_DEDENT { FLOC(@$, @2); LLOC(@$,@2); $$ = $2; } ; sep_statements - : sep statements { @$.first = @2.first; $$ = $2; } - | type_ignore_sep statements { @$.first = @2.first; $$ = $2; } + : sep statements { FLOC(@$, @2); $$ = $2; } + | type_ignore_sep statements { FLOC(@$, @2); $$ = $2; } ; body_stmts @@ -700,7 +700,7 @@ function_def : decorators_opt KW_DEF id "(" parameter_list_opt ")" ":" body_stmts { $$ = FUNCTION_01($1, $3, $5, $8, @$); } | decorators_opt KW_DEF id "(" parameter_list_opt ")" "->" expr ":" - body_stmts { @$.first = @2.first; $$ = FUNCTION_02($1, $3, $5, $8, $10, @$); } + body_stmts { FLOC(@$, @2); $$ = FUNCTION_02($1, $3, $5, $8, $10, @$); } | decorators_opt KW_DEF id "(" parameter_list_opt ")" ":" TK_TYPE_COMMENT sep statements { $$ = FUNCTION_03($1, $3, $5, $10, $8, @$); } @@ -719,7 +719,7 @@ class_def : decorators_opt KW_CLASS id ":" body_stmts { $$ = CLASS_01($1, $3, $5, @$); } | decorators_opt KW_CLASS id "(" call_arguement_list ")" ":" body_stmts { - @$.first = @2.first; $$ = CLASS_02($1, $3, $5, $8, @$); } + FLOC(@$, @2); $$ = CLASS_02($1, $3, $5, $8, @$); } ; async_func_def diff --git a/src/lpython/parser/semantics.h b/src/lpython/parser/semantics.h index 84fe98492e..8fcc8e06e6 100644 --- a/src/lpython/parser/semantics.h +++ b/src/lpython/parser/semantics.h @@ -58,7 +58,8 @@ static inline T** vec_cast(const Vec &x) { return s; } -// Assign last location to `a` from `b` +// Assign first/last location to `a` from `b` +#define FLOC(a, b) a.first = b.first; #define LLOC(a, b) a.last = b.last; From d5ce5644d3c9effeb27a6d544fcef0aa20b2efc5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ond=C5=99ej=20=C4=8Cert=C3=ADk?= Date: Fri, 23 Sep 2022 14:42:43 -0400 Subject: [PATCH 8/8] Put FLOC at all functions --- src/lpython/parser/parser.yy | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/lpython/parser/parser.yy b/src/lpython/parser/parser.yy index c5fe2b7f50..c61b84be4e 100644 --- a/src/lpython/parser/parser.yy +++ b/src/lpython/parser/parser.yy @@ -698,26 +698,26 @@ comma_opt function_def : decorators_opt KW_DEF id "(" parameter_list_opt ")" ":" - body_stmts { $$ = FUNCTION_01($1, $3, $5, $8, @$); } + body_stmts { FLOC(@$, @2); $$ = FUNCTION_01($1, $3, $5, $8, @$); } | decorators_opt KW_DEF id "(" parameter_list_opt ")" "->" expr ":" body_stmts { FLOC(@$, @2); $$ = FUNCTION_02($1, $3, $5, $8, $10, @$); } | decorators_opt KW_DEF id "(" parameter_list_opt ")" ":" TK_TYPE_COMMENT sep statements { - $$ = FUNCTION_03($1, $3, $5, $10, $8, @$); } + FLOC(@$, @2); $$ = FUNCTION_03($1, $3, $5, $10, $8, @$); } | decorators_opt KW_DEF id "(" parameter_list_opt ")" "->" expr ":" TK_TYPE_COMMENT sep statements { - $$ = FUNCTION_04($1, $3, $5, $8, $12, $10, @$); } + FLOC(@$, @2); $$ = FUNCTION_04($1, $3, $5, $8, $12, $10, @$); } | decorators_opt KW_DEF id "(" parameter_list_opt ")" ":" sep TK_TYPE_COMMENT sep statements { - $$ = FUNCTION_03($1, $3, $5, $11, $9, @$); } + FLOC(@$, @2); $$ = FUNCTION_03($1, $3, $5, $11, $9, @$); } | decorators_opt KW_DEF id "(" parameter_list_opt ")" "->" expr ":" sep TK_TYPE_COMMENT sep statements { - $$ = FUNCTION_04($1, $3, $5, $8, $13, $11, @$); } + FLOC(@$, @2); $$ = FUNCTION_04($1, $3, $5, $8, $13, $11, @$); } ; class_def : decorators_opt KW_CLASS id ":" body_stmts { - $$ = CLASS_01($1, $3, $5, @$); } + FLOC(@$, @2); $$ = CLASS_01($1, $3, $5, @$); } | decorators_opt KW_CLASS id "(" call_arguement_list ")" ":" body_stmts { FLOC(@$, @2); $$ = CLASS_02($1, $3, $5, $8, @$); } ;