Skip to content

Fix some parser related issues #1141

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 5 commits into from
Sep 23, 2022
Merged

Fix some parser related issues #1141

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
680cf04
Parse tuple's in AugAssignment
Thirumalai-Shaktivel Sep 23, 2022
c89c281
Recognize Type_Ignore inside the parentheses of the function call
Thirumalai-Shaktivel Sep 23, 2022
3c6bc7e
Fix tuple issue in the subscript slice
Thirumalai-Shaktivel Sep 23, 2022
c6a6648
Store None for the default kwonlyargs in function definition
Thirumalai-Shaktivel Sep 23, 2022
64ef2fa
Add the tests and update refs.
Thirumalai-Shaktivel Sep 23, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 5 additions & 2 deletions src/lpython/parser/parser.yy
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -442,7 +442,7 @@ assignment_statement
;

augassign_statement
: expr augassign_op expr { $$ = AUGASSIGN_01($1, $2, $3, @$); }
: expr augassign_op tuple_list { $$ = AUGASSIGN_01($1, $2, $3, @$); }
;

augassign_op
Expand Down Expand Up @@ -879,6 +879,8 @@ primary

function_call
: primary "(" call_arguement_list ")" { $$ = CALL_01($1, $3, @$); }
| primary "(" TK_TYPE_IGNORE call_arguement_list ")" {
$$ = CALL_01($1, $4, @$); extract_type_comment(p, @$, $3); }
| primary "(" expr comp_for_items ")" {
$$ = CALL_02($1, A2LIST(p.m_a, GENERATOR_EXPR($3, $4, @$)), @$); }
| function_call "(" call_arguement_list ")" { $$ = CALL_01($1, $3, @$); }
Expand Down Expand Up @@ -912,7 +914,8 @@ slice_items
;

slice_item
: slice_item_list comma_opt { $$ = TUPLE($1, @$); }
: slice_item_list { $$ = TUPLE_01($1, @$); }
| slice_item_list "," { $$ = TUPLE_03($1, @$); }
;

subscript
Expand Down
37 changes: 17 additions & 20 deletions src/lpython/parser/semantics.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -367,10 +367,17 @@ Arg** ARG2LIST(Allocator &al, Arg *x) {
if(n_##x > 0) { \
for(size_t i = 0; i < n_##x; i++) { \
_m_##x.push_back(al, m_##x[i]->_arg); \
if(m_##x[i]->default_value && !kw) { \
defaults.push_back(al, m_##x[i]->defaults); \
} else if (m_##x[i]->default_value){ \
kw_defaults.push_back(al, m_##x[i]->defaults); \
if(m_##x[i]->default_value) { \
if (kw == 0) { \
defaults.push_back(al, m_##x[i]->defaults); \
} else { \
kw_defaults.push_back(al, m_##x[i]->defaults); \
} \
} else { \
if (kw == 1) { \
kw_defaults.push_back(al, \
(expr_t*)make_ConstantNone_t(al, l, nullptr)); \
} \
} \
} \
r->arguments.m_##x = _m_##x.p; \
Expand All @@ -394,25 +401,25 @@ static inline Args *FUNC_ARGS_01(Allocator &al, Location &l, Fn_Arg *parameters)
if(parameters != nullptr) {
Arg** m_posonlyargs = parameters->posonlyargs.p;
size_t n_posonlyargs = parameters->posonlyargs.n;
FUNC_ARGS_(posonlyargs, false);
FUNC_ARGS_(posonlyargs, 0);
}
if(parameters != nullptr && parameters->args_val) {
Arg** m_args = parameters->args->args.p;
size_t n_args = parameters->args->args.n;
FUNC_ARGS_(args, false);
FUNC_ARGS_(args, 0);

if(parameters->args->var_kw_val) {
Arg** m_vararg = parameters->args->var_kw->vararg.p;
size_t n_vararg = parameters->args->var_kw->vararg.n;
FUNC_ARGS_(vararg, false);
FUNC_ARGS_(vararg, 0);

Arg** m_kwonlyargs = parameters->args->var_kw->kwonlyargs.p;
size_t n_kwonlyargs = parameters->args->var_kw->kwonlyargs.n;
FUNC_ARGS_(kwonlyargs, true);
FUNC_ARGS_(kwonlyargs, 1);

Arg** m_kwarg = parameters->args->var_kw->kwarg.p;
size_t n_kwarg = parameters->args->var_kw->kwarg.n;
FUNC_ARGS_(kwarg, true);
FUNC_ARGS_(kwarg, 2);
}
}
r->arguments.m_kw_defaults = kw_defaults.p;
Expand Down Expand Up @@ -905,22 +912,12 @@ static inline ast_t* ID_TUPLE_02(Allocator &al, Location &l, Vec<ast_t*> elts) {
#define COMP_EXPR_1(expr, generators, l) make_GeneratorExp_t(p.m_a, l, \
EXPR(expr), generators.p, generators.n)

expr_t* CHECK_TUPLE(expr_t *x) {
if(is_a<Tuple_t>(*x) && down_cast<Tuple_t>(x)->n_elts == 1) {
return down_cast<Tuple_t>(x)->m_elts[0];
} else {
return x;
}
}

#define ELLIPSIS(l) make_ConstantEllipsis_t(p.m_a, l, nullptr)

#define NONE(l) make_ConstantNone_t(p.m_a, l, nullptr)

#define TUPLE(elts, l) make_Tuple_t(p.m_a, l, \
EXPRS(elts), elts.size(), expr_contextType::Load)
#define SUBSCRIPT_01(value, slice, l) make_Subscript_t(p.m_a, l, \
EXPR(value), CHECK_TUPLE(EXPR(slice)), expr_contextType::Load)
EXPR(value), EXPR(slice), expr_contextType::Load)

static inline ast_t* SLICE(Allocator &al, Location &l,
ast_t *lower, ast_t *upper, ast_t *_step) {
Expand Down
2 changes: 2 additions & 0 deletions tests/parser/statements1.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,8 @@
obj = obj,

x += 1
x += y, z
x += (y, z),

x: i64
y: i32 = 1
Expand Down
3 changes: 3 additions & 0 deletions tests/parser/type_comment1.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -32,3 +32,6 @@ def main():

from sympy.simplify import (collect, powsimp, # type: ignore
separatevars, simplify)

await test( # type: ignore
x, y, z)
2 changes: 1 addition & 1 deletion tests/reference/ast_new-function_def2-52c4587.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@
"outfile": null,
"outfile_hash": null,
"stdout": "ast_new-function_def2-52c4587.stdout",
"stdout_hash": "f94b525c239bede5185c86dbf285f82e9a4ec3708e307c191f6270eb",
"stdout_hash": "487f9f6f7ee1f07bfceae82a4dcff7b35d1861d587cd1f1f5d156912",
"stderr": null,
"stderr_hash": null,
"returncode": 0
Expand Down
2 changes: 1 addition & 1 deletion tests/reference/ast_new-function_def2-52c4587.stdout
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1 +1 @@
(Module [(FunctionDef test_01 ([] [] [] [] [] [] []) [(Pass)] [] () ()) (FunctionDef test_02 ([] [] [(x (Name i32 Load) ())] [] [] [] []) [(Pass)] [] () ()) (FunctionDef test_03 ([] [] [(x () ())] [(y () ()) (z () ())] [] [] []) [(Pass)] [] () ()) (FunctionDef test_04 ([] [] [(x () ())] [(y () ()) (z () ())] [] [(args () ())] []) [(Pass)] [] () ()) (FunctionDef test_05 ([] [] [(x () ())] [] [] [(args () ())] []) [(Pass)] [] () ()) (FunctionDef test_06 ([] [] [] [] [] [(args () ())] []) [(Pass)] [] () ()) (FunctionDef test_07 ([] [(x (Name i32 Load) ()) (y () ())] [] [] [] [] []) [(Pass)] [] () ()) (FunctionDef test_08 ([] [(x () ())] [(y () ())] [] [] [] []) [(Pass)] [] () ()) (FunctionDef test_09 ([] [(x () ())] [(y () ())] [(z () ())] [] [] []) [(Pass)] [] () ()) (FunctionDef test_10 ([] [(x () ())] [(y () ())] [(z () ())] [] [(args (Name i32 Load) ())] []) [(Pass)] [] () ()) (FunctionDef test_11 ([] [(x () ())] [(y () ())] [] [] [(args () ())] []) [(Pass)] [] () ()) (FunctionDef test_12 ([] [(x () ())] [] [] [] [(args () ())] []) [(Pass)] [] () ()) (FunctionDef test_13 ([] [(x (Name i32 Load) ()) (y (Name i32 Load) ())] [] [] [] [] [(ConstantInt 1 ()) (ConstantInt 1 ())]) [(Pass)] [] () ()) (FunctionDef test_14 ([] [(x () ())] [(y () ())] [(z () ())] [] [(args (Name i32 Load) ())] []) [(Pass)] [] (Name i32 Load) ()) (FunctionDef test_15 ([(a () ())] [] [] [] [] [] []) [(Pass)] [] () ()) (FunctionDef test_16 ([(a () ())] [(b () ()) (c () ())] [] [] [] [] []) [(Pass)] [] () ()) (FunctionDef test_17 ([(a () ())] [] [(b () ())] [] [] [] []) [(Pass)] [] () ()) (FunctionDef test_18 ([(a (Name i32 Load) ())] [] [(b (Name i64 Load) ())] [(c (Name i32 Load) ()) (d (Name i32 Load) ())] [] [] []) [(Pass)] [] () ()) (FunctionDef test_19 ([(a () ())] [] [(b () ())] [(c () ())] [] [(d () ())] []) [(Pass)] [] () ()) (FunctionDef test_20 ([(a () ())] [] [] [] [] [(b () ())] []) [(Pass)] [] () ()) (FunctionDef test_21 ([(a () ())] [] [(b () ())] [] [] [(c () ())] []) [(Pass)] [] () ()) (FunctionDef test_22 ([(a () ())] [(b () ()) (c () ())] [(d () ())] [] [] [] []) [(Pass)] [] () ()) (FunctionDef test_23 ([(a () ())] [(b () ()) (c () ())] [] [] [] [(d () ())] []) [(Pass)] [] () ()) (FunctionDef test_24 ([(a () ())] [(b () ()) (c () ())] [(d () ())] [] [] [(e () ())] []) [(Pass)] [] () ()) (FunctionDef test_25 ([(a () ())] [(b () ()) (c () ())] [(d () ())] [(e () ())] [] [(f () ())] []) [(Pass)] [] () ()) (FunctionDef test_26 ([] [] [] [(a () ())] [] [] []) [(Pass)] [] () ()) (FunctionDef test_27 ([] [] [] [(a () ())] [] [(b () ())] []) [(Pass)] [] () ()) (FunctionDef test_28 ([] [(a () ())] [] [(b () ())] [] [] []) [(Pass)] [] () ()) (FunctionDef test_29 ([] [(a () ())] [] [(b () ())] [] [(c () ())] []) [(Pass)] [] () ()) (FunctionDef test_30 ([(a () ())] [] [] [(b () ())] [] [] []) [(Pass)] [] () ()) (FunctionDef test_31 ([(a () ())] [(b () ())] [] [(c () ())] [] [] []) [(Pass)] [] () ()) (FunctionDef test_32 ([(a () ())] [] [] [(c () ())] [] [(d () ())] []) [(Pass)] [] () ()) (FunctionDef test_33 ([(a () ())] [(b () ())] [] [(c () ())] [] [(d () ())] []) [(Pass)] [] () ()) (Expr (Call (Name test Load) [] [])) (Expr (Call (Name test Load) [(Name x Load) (Name y Load)] [])) (Expr (Call (Name test Load) [(Name x Load)] [(y (ConstantInt 1 ())) (z (ConstantStr "123" ()))])) (Expr (Call (Name test Load) [(ConstantInt 100 ())] [(() (Name x Load))])) (Expr (Call (Name test Load) [(Starred (Name x Load) Load)] [(() (Name y Load))])) (Expr (Call (Name test Load) [(Starred (Name x Load) Load) (Name y Load)] [])) (Expr (Call (Name test Load) [(Starred (Name x Load) Load) (Starred (Name y Load) Load)] [])) (Expr (Call (Name test Load) [] [(() (Name x Load)) (() (Name y Load))])) (Expr (Call (Name test Load) [(Starred (Name x Load) Load)] [(y (ConstantInt 1 ())) (z (ConstantInt 2 ())) (() (Name a Load)) (b (ConstantInt 1 ()))])) (Expr (Call (Name test Load) [(Starred (Name x Load) Load)] [(() (Name a Load)) (b (ConstantStr "1" ()))])) (Expr (Call (Name test Load) [(Starred (Name y Load) Load)] [(x (ConstantInt 1 ())) (z (ConstantInt 2 ())) (() (Name a Load)) (b (ConstantStr "1" ()))])) (Expr (Call (Attribute (Name lp Load) test Load) [] [])) (Expr (Call (Attribute (Name lp Load) test Load) [(Name x Load) (Name y Load)] [])) (Expr (Call (Attribute (Name lp Load) test Load) [(Name x Load)] [(y (ConstantInt 1 ())) (z (ConstantStr "123" ()))])) (Expr (Subscript (Call (Name test Load) [] []) (ConstantStr "version" ()) Load)) (Expr (Subscript (Call (Name test Load) [(Name x Load) (Name y Load)] []) (ConstantStr "version" ()) Load)) (Expr (Subscript (Call (Name test Load) [(Name x Load) (Starred (Name y Load) Load)] []) (Slice () (UnaryOp USub (ConstantInt 1 ())) ()) Load)) (Expr (Subscript (Subscript (Call (Name test Load) [] []) (ConstantInt 1 ()) Load) (ConstantInt 1 ()) Load)) (Expr (Call (Subscript (Name x Load) (ConstantStr "numpystr" ()) Load) [] [])) (Expr (Call (Subscript (Name x Load) (ConstantStr "int" ()) Load) [] [])) (Expr (Call (Subscript (Name x Load) (Name func Load) Load) [(Starred (Name args Load) Load)] [(() (Name kwargs Load))])) (Expr (Call (Subscript (Name test Load) (ConstantInt 0 ()) Load) [(Starred (Subscript (Name test Load) (Slice (ConstantInt 1 ()) () ()) Load) Load)] [])) (Expr (Call (BinOp (Name obj Load) Mult (Attribute (Attribute (Name self Load) _arr Load) ndim Load)) [(Starred (Attribute (Attribute (Name self Load) _arr Load) shape Load) Load)] [])) (Expr (Call (Name traverse Load) [(Tuple [(Name index Load) (Name value Load)] Load) (Name visit Load) (Starred (Name args Load) Load)] [(result (Name result Load)) (() (Name kwargs Load))]))] [])
(Module [(FunctionDef test_01 ([] [] [] [] [] [] []) [(Pass)] [] () ()) (FunctionDef test_02 ([] [] [(x (Name i32 Load) ())] [] [] [] []) [(Pass)] [] () ()) (FunctionDef test_03 ([] [] [(x () ())] [(y () ()) (z () ())] [(ConstantNone ()) (ConstantNone ())] [] []) [(Pass)] [] () ()) (FunctionDef test_04 ([] [] [(x () ())] [(y () ()) (z () ())] [(ConstantNone ()) (ConstantNone ())] [(args () ())] []) [(Pass)] [] () ()) (FunctionDef test_05 ([] [] [(x () ())] [] [] [(args () ())] []) [(Pass)] [] () ()) (FunctionDef test_06 ([] [] [] [] [] [(args () ())] []) [(Pass)] [] () ()) (FunctionDef test_07 ([] [(x (Name i32 Load) ()) (y () ())] [] [] [] [] []) [(Pass)] [] () ()) (FunctionDef test_08 ([] [(x () ())] [(y () ())] [] [] [] []) [(Pass)] [] () ()) (FunctionDef test_09 ([] [(x () ())] [(y () ())] [(z () ())] [(ConstantNone ())] [] []) [(Pass)] [] () ()) (FunctionDef test_10 ([] [(x () ())] [(y () ())] [(z () ())] [(ConstantNone ())] [(args (Name i32 Load) ())] []) [(Pass)] [] () ()) (FunctionDef test_11 ([] [(x () ())] [(y () ())] [] [] [(args () ())] []) [(Pass)] [] () ()) (FunctionDef test_12 ([] [(x () ())] [] [] [] [(args () ())] []) [(Pass)] [] () ()) (FunctionDef test_13 ([] [(x (Name i32 Load) ()) (y (Name i32 Load) ())] [] [] [] [] [(ConstantInt 1 ()) (ConstantInt 1 ())]) [(Pass)] [] () ()) (FunctionDef test_14 ([] [(x () ())] [(y () ())] [(z () ())] [(ConstantNone ())] [(args (Name i32 Load) ())] []) [(Pass)] [] (Name i32 Load) ()) (FunctionDef test_15 ([(a () ())] [] [] [] [] [] []) [(Pass)] [] () ()) (FunctionDef test_16 ([(a () ())] [(b () ()) (c () ())] [] [] [] [] []) [(Pass)] [] () ()) (FunctionDef test_17 ([(a () ())] [] [(b () ())] [] [] [] []) [(Pass)] [] () ()) (FunctionDef test_18 ([(a (Name i32 Load) ())] [] [(b (Name i64 Load) ())] [(c (Name i32 Load) ()) (d (Name i32 Load) ())] [(ConstantNone ()) (ConstantNone ())] [] []) [(Pass)] [] () ()) (FunctionDef test_19 ([(a () ())] [] [(b () ())] [(c () ())] [(ConstantNone ())] [(d () ())] []) [(Pass)] [] () ()) (FunctionDef test_20 ([(a () ())] [] [] [] [] [(b () ())] []) [(Pass)] [] () ()) (FunctionDef test_21 ([(a () ())] [] [(b () ())] [] [] [(c () ())] []) [(Pass)] [] () ()) (FunctionDef test_22 ([(a () ())] [(b () ()) (c () ())] [(d () ())] [] [] [] []) [(Pass)] [] () ()) (FunctionDef test_23 ([(a () ())] [(b () ()) (c () ())] [] [] [] [(d () ())] []) [(Pass)] [] () ()) (FunctionDef test_24 ([(a () ())] [(b () ()) (c () ())] [(d () ())] [] [] [(e () ())] []) [(Pass)] [] () ()) (FunctionDef test_25 ([(a () ())] [(b () ()) (c () ())] [(d () ())] [(e () ())] [(ConstantNone ())] [(f () ())] []) [(Pass)] [] () ()) (FunctionDef test_26 ([] [] [] [(a () ())] [(ConstantNone ())] [] []) [(Pass)] [] () ()) (FunctionDef test_27 ([] [] [] [(a () ())] [(ConstantNone ())] [(b () ())] []) [(Pass)] [] () ()) (FunctionDef test_28 ([] [(a () ())] [] [(b () ())] [(ConstantNone ())] [] []) [(Pass)] [] () ()) (FunctionDef test_29 ([] [(a () ())] [] [(b () ())] [(ConstantNone ())] [(c () ())] []) [(Pass)] [] () ()) (FunctionDef test_30 ([(a () ())] [] [] [(b () ())] [(ConstantNone ())] [] []) [(Pass)] [] () ()) (FunctionDef test_31 ([(a () ())] [(b () ())] [] [(c () ())] [(ConstantNone ())] [] []) [(Pass)] [] () ()) (FunctionDef test_32 ([(a () ())] [] [] [(c () ())] [(ConstantNone ())] [(d () ())] []) [(Pass)] [] () ()) (FunctionDef test_33 ([(a () ())] [(b () ())] [] [(c () ())] [(ConstantNone ())] [(d () ())] []) [(Pass)] [] () ()) (Expr (Call (Name test Load) [] [])) (Expr (Call (Name test Load) [(Name x Load) (Name y Load)] [])) (Expr (Call (Name test Load) [(Name x Load)] [(y (ConstantInt 1 ())) (z (ConstantStr "123" ()))])) (Expr (Call (Name test Load) [(ConstantInt 100 ())] [(() (Name x Load))])) (Expr (Call (Name test Load) [(Starred (Name x Load) Load)] [(() (Name y Load))])) (Expr (Call (Name test Load) [(Starred (Name x Load) Load) (Name y Load)] [])) (Expr (Call (Name test Load) [(Starred (Name x Load) Load) (Starred (Name y Load) Load)] [])) (Expr (Call (Name test Load) [] [(() (Name x Load)) (() (Name y Load))])) (Expr (Call (Name test Load) [(Starred (Name x Load) Load)] [(y (ConstantInt 1 ())) (z (ConstantInt 2 ())) (() (Name a Load)) (b (ConstantInt 1 ()))])) (Expr (Call (Name test Load) [(Starred (Name x Load) Load)] [(() (Name a Load)) (b (ConstantStr "1" ()))])) (Expr (Call (Name test Load) [(Starred (Name y Load) Load)] [(x (ConstantInt 1 ())) (z (ConstantInt 2 ())) (() (Name a Load)) (b (ConstantStr "1" ()))])) (Expr (Call (Attribute (Name lp Load) test Load) [] [])) (Expr (Call (Attribute (Name lp Load) test Load) [(Name x Load) (Name y Load)] [])) (Expr (Call (Attribute (Name lp Load) test Load) [(Name x Load)] [(y (ConstantInt 1 ())) (z (ConstantStr "123" ()))])) (Expr (Subscript (Call (Name test Load) [] []) (ConstantStr "version" ()) Load)) (Expr (Subscript (Call (Name test Load) [(Name x Load) (Name y Load)] []) (ConstantStr "version" ()) Load)) (Expr (Subscript (Call (Name test Load) [(Name x Load) (Starred (Name y Load) Load)] []) (Slice () (UnaryOp USub (ConstantInt 1 ())) ()) Load)) (Expr (Subscript (Subscript (Call (Name test Load) [] []) (ConstantInt 1 ()) Load) (ConstantInt 1 ()) Load)) (Expr (Call (Subscript (Name x Load) (ConstantStr "numpystr" ()) Load) [] [])) (Expr (Call (Subscript (Name x Load) (ConstantStr "int" ()) Load) [] [])) (Expr (Call (Subscript (Name x Load) (Name func Load) Load) [(Starred (Name args Load) Load)] [(() (Name kwargs Load))])) (Expr (Call (Subscript (Name test Load) (ConstantInt 0 ()) Load) [(Starred (Subscript (Name test Load) (Slice (ConstantInt 1 ()) () ()) Load) Load)] [])) (Expr (Call (BinOp (Name obj Load) Mult (Attribute (Attribute (Name self Load) _arr Load) ndim Load)) [(Starred (Attribute (Attribute (Name self Load) _arr Load) shape Load) Load)] [])) (Expr (Call (Name traverse Load) [(Tuple [(Name index Load) (Name value Load)] Load) (Name visit Load) (Starred (Name args Load) Load)] [(result (Name result Load)) (() (Name kwargs Load))]))] [])
2 changes: 1 addition & 1 deletion tests/reference/ast_new-function_def3-f66064a.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@
"outfile": null,
"outfile_hash": null,
"stdout": "ast_new-function_def3-f66064a.stdout",
"stdout_hash": "6eadb50ca8a83c6fb88e758234d91cceab17ee0177d97fc2bc04d503",
"stdout_hash": "926e7c2dba8ad4f0536cb81c654f3ace4d4edce1b719da3dccc35f51",
"stderr": null,
"stderr_hash": null,
"returncode": 0
Expand Down
Loading