From c38341d75205b06df4d07f393e29f9612d31d275 Mon Sep 17 00:00:00 2001 From: Sehoon Kim Date: Sat, 2 Jul 2022 16:09:13 +0900 Subject: [PATCH 1/5] checking Eof before calling parseIdent --- src/res_core.ml | 25 +++++++++++++++++++++---- 1 file changed, 21 insertions(+), 4 deletions(-) diff --git a/src/res_core.ml b/src/res_core.ml index 31258253..f7c6f52d 100644 --- a/src/res_core.ml +++ b/src/res_core.ml @@ -600,7 +600,11 @@ let parseHashIdent ~startPos p = in Parser.next p; (i, mkLoc startPos p.prevEndPos) - | _ -> parseIdent ~startPos ~msg:ErrorMessages.variantIdent p + | token -> + if token = Eof then ( + Parser.err ~startPos p (Diagnostics.unexpected token p.breadcrumbs); + ("", mkLoc startPos p.prevEndPos)) + else parseIdent ~startPos ~msg:ErrorMessages.variantIdent p (* Ldot (Ldot (Lident "Foo", "Bar"), "baz") *) let parseValuePath p = @@ -1090,7 +1094,12 @@ let rec parsePattern ?(alias = true) ?(or_ = true) p = in Parser.next p; (i, mkLoc startPos p.prevEndPos) - | _ -> parseIdent ~msg:ErrorMessages.variantIdent ~startPos p + | token -> + if token = Eof then ( + Parser.err ~startPos p + (Diagnostics.unexpected token p.breadcrumbs); + ("", mkLoc startPos p.prevEndPos)) + else parseIdent ~msg:ErrorMessages.variantIdent ~startPos p in match p.Parser.token with | Lparen -> parseVariantPatternArgs p ident startPos attrs @@ -3808,7 +3817,11 @@ and parseAtomicTypExpr ~attrs p = | SingleQuote -> Parser.next p; let ident, loc = - parseIdent ~msg:ErrorMessages.typeVar ~startPos:p.startPos p + if p.Parser.token = Eof then ( + Parser.err ~startPos:p.startPos p + (Diagnostics.unexpected p.Parser.token p.breadcrumbs); + ("", mkLoc p.startPos p.prevEndPos)) + else parseIdent ~msg:ErrorMessages.typeVar ~startPos:p.startPos p in Ast_helper.Typ.var ~loc ~attrs ident | Underscore -> @@ -4596,7 +4609,11 @@ and parseTypeParam p = | SingleQuote -> Parser.next p; let ident, loc = - parseIdent ~msg:ErrorMessages.typeParam ~startPos:p.startPos p + if p.Parser.token = Eof then ( + Parser.err ~startPos:p.startPos p + (Diagnostics.unexpected p.Parser.token p.breadcrumbs); + ("", mkLoc p.startPos p.prevEndPos)) + else parseIdent ~msg:ErrorMessages.typeParam ~startPos:p.startPos p in Some (Ast_helper.Typ.var ~loc ident, variance) | Underscore -> From f01534ee0dc829003479e2d5b6f048f3268e7502 Mon Sep 17 00:00:00 2001 From: Sehoon Kim Date: Sat, 2 Jul 2022 21:28:12 +0900 Subject: [PATCH 2/5] check Eof before calling Parser.next --- src/res_core.ml | 7 +++++-- tests/parsing/errors/other/expected/for.res.txt | 12 ++++++++++++ tests/parsing/errors/other/for.res | 1 + .../errors/typeDef/expected/keywordOnly.res.txt | 9 +++++++++ tests/parsing/errors/typeDef/keywordOnly.res | 1 + 5 files changed, 28 insertions(+), 2 deletions(-) create mode 100644 tests/parsing/errors/other/expected/for.res.txt create mode 100644 tests/parsing/errors/other/for.res create mode 100644 tests/parsing/errors/typeDef/expected/keywordOnly.res.txt create mode 100644 tests/parsing/errors/typeDef/keywordOnly.res diff --git a/src/res_core.ml b/src/res_core.ml index f7c6f52d..e10360d4 100644 --- a/src/res_core.ml +++ b/src/res_core.ml @@ -643,7 +643,7 @@ let parseValuePath p = res | token -> Parser.err p (Diagnostics.unexpected token p.breadcrumbs); - Parser.next p; + if token <> Eof then Parser.next p; Longident.Lident "_" in Location.mkloc ident (mkLoc startPos p.prevEndPos) @@ -3258,7 +3258,10 @@ and parseForRest hasOpeningParen pattern startPos p = Parser.err p (Diagnostics.unexpected token p.breadcrumbs); Asttypes.Upto in - Parser.next p; + if p.Parser.token = Eof then + Parser.err ~startPos:p.startPos p + (Diagnostics.unexpected p.Parser.token p.breadcrumbs) + else Parser.next p; let e2 = parseExpr ~context:WhenExpr p in if hasOpeningParen then Parser.expect Rparen p; Parser.expect Lbrace p; diff --git a/tests/parsing/errors/other/expected/for.res.txt b/tests/parsing/errors/other/expected/for.res.txt new file mode 100644 index 00000000..de41bea0 --- /dev/null +++ b/tests/parsing/errors/other/expected/for.res.txt @@ -0,0 +1,12 @@ + + Syntax error! + tests/parsing/errors/other/for.res:1:6-2:0 + + 1 │ for x + 2 │ + + Did you forget a `in` here? + +;;for x = [%rescript.exprhole ] to [%rescript.exprhole ] do + [%rescript.exprhole ] + done \ No newline at end of file diff --git a/tests/parsing/errors/other/for.res b/tests/parsing/errors/other/for.res new file mode 100644 index 00000000..5c8c39ba --- /dev/null +++ b/tests/parsing/errors/other/for.res @@ -0,0 +1 @@ +for x diff --git a/tests/parsing/errors/typeDef/expected/keywordOnly.res.txt b/tests/parsing/errors/typeDef/expected/keywordOnly.res.txt new file mode 100644 index 00000000..e15e02ad --- /dev/null +++ b/tests/parsing/errors/typeDef/expected/keywordOnly.res.txt @@ -0,0 +1,9 @@ + + Syntax error! + tests/parsing/errors/typeDef/keywordOnly.res:1:5 + + 1 │ type + + I'm not sure what to parse here when looking at "eof". + +type nonrec _ \ No newline at end of file diff --git a/tests/parsing/errors/typeDef/keywordOnly.res b/tests/parsing/errors/typeDef/keywordOnly.res new file mode 100644 index 00000000..c3dc0520 --- /dev/null +++ b/tests/parsing/errors/typeDef/keywordOnly.res @@ -0,0 +1 @@ +type \ No newline at end of file From c8608dd6e3e854de2d0d497541ab439e90697af4 Mon Sep 17 00:00:00 2001 From: Sehoon Kim Date: Sun, 3 Jul 2022 17:18:26 +0900 Subject: [PATCH 3/5] add guard for Parser.next --- src/res_core.ml | 66 +++++++++++++++++++++++++++++-------------------- 1 file changed, 39 insertions(+), 27 deletions(-) diff --git a/src/res_core.ml b/src/res_core.ml index e10360d4..f9b0954b 100644 --- a/src/res_core.ml +++ b/src/res_core.ml @@ -556,10 +556,14 @@ let rec parseLident p = Parser.next p; let loc = mkLoc startPos p.prevEndPos in (ident, loc) - | _ -> ( - match recoverLident p with - | Some () -> parseLident p - | None -> ("_", mkLoc startPos p.prevEndPos)) + | token -> ( + if token = Eof then ( + Parser.err ~startPos p (Diagnostics.unexpected token p.breadcrumbs); + ("_", mkLoc startPos p.prevEndPos)) + else + match recoverLident p with + | Some () -> parseLident p + | None -> ("_", mkLoc startPos p.prevEndPos)) let parseIdent ~msg ~startPos p = match p.Parser.token with @@ -639,11 +643,11 @@ let parseValuePath p = Parser.err p (Diagnostics.unexpected p.Parser.token p.breadcrumbs); Longident.Lident ident) in - if p.token <> Eof then Parser.next p; + Parser.nextUnsafe p; res | token -> Parser.err p (Diagnostics.unexpected token p.breadcrumbs); - if token <> Eof then Parser.next p; + Parser.nextUnsafe p; Longident.Lident "_" in Location.mkloc ident (mkLoc startPos p.prevEndPos) @@ -830,7 +834,7 @@ let parseConstant p = Parser.err p (Diagnostics.unexpected token p.breadcrumbs); Pconst_string ("", None) in - Parser.next p; + Parser.nextUnsafe p; constant let parseTemplateConstant ~prefix (p : Parser.t) = @@ -1124,11 +1128,14 @@ let rec parsePattern ?(alias = true) ?(or_ = true) p = Ast_helper.Pat.extension ~loc ~attrs extension | token -> ( Parser.err p (Diagnostics.unexpected token p.breadcrumbs); - match - skipTokensAndMaybeRetry p ~isStartOfGrammar:Grammar.isAtomicPatternStart - with - | None -> Recover.defaultPattern () - | Some () -> parsePattern p) + if p.Parser.token = Eof then Recover.defaultPattern () + else + match + skipTokensAndMaybeRetry p + ~isStartOfGrammar:Grammar.isAtomicPatternStart + with + | None -> Recover.defaultPattern () + | Some () -> parsePattern p) in let pat = if alias then parseAliasPattern ~attrs pat p else pat in if or_ then parseOrPattern pat p else pat @@ -1871,11 +1878,13 @@ and parseAtomicExpr p = | token -> ( let errPos = p.prevEndPos in Parser.err ~startPos:errPos p (Diagnostics.unexpected token p.breadcrumbs); - match - skipTokensAndMaybeRetry p ~isStartOfGrammar:Grammar.isAtomicExprStart - with - | None -> Recover.defaultExpr () - | Some () -> parseAtomicExpr p) + if p.Parser.token = Eof then Recover.defaultExpr () + else + match + skipTokensAndMaybeRetry p ~isStartOfGrammar:Grammar.isAtomicExprStart + with + | None -> Recover.defaultExpr () + | Some () -> parseAtomicExpr p) in Parser.eatBreadcrumb p; expr @@ -1906,7 +1915,7 @@ and parseFirstClassModuleExpr ~startPos p = and parseBracketAccess p expr startPos = Parser.leaveBreadcrumb p Grammar.ExprArrayAccess; let lbracket = p.startPos in - Parser.next p; + Parser.expect Lbracket p; let stringStart = p.startPos in match p.Parser.token with | String s -> ( @@ -3619,7 +3628,7 @@ and parseValueOrConstructor p = Ast_helper.Exp.ident ~loc (Location.mkloc lident loc) | token -> if acc = [] then ( - Parser.next p; + Parser.nextUnsafe p; Parser.err p (Diagnostics.unexpected token p.breadcrumbs); Recover.defaultExpr ()) else @@ -3872,14 +3881,17 @@ and parseAtomicTypExpr ~attrs p = | Lbrace -> parseRecordOrObjectType ~attrs p | token -> ( Parser.err p (Diagnostics.unexpected token p.breadcrumbs); - match - skipTokensAndMaybeRetry p ~isStartOfGrammar:Grammar.isAtomicTypExprStart - with - | Some () -> parseAtomicTypExpr ~attrs p - | None -> - Parser.err ~startPos:p.prevEndPos p - (Diagnostics.unexpected token p.breadcrumbs); - Recover.defaultType ()) + if p.Parser.token = Eof then Recover.defaultType () + else + match + skipTokensAndMaybeRetry p + ~isStartOfGrammar:Grammar.isAtomicTypExprStart + with + | Some () -> parseAtomicTypExpr ~attrs p + | None -> + Parser.err ~startPos:p.prevEndPos p + (Diagnostics.unexpected token p.breadcrumbs); + Recover.defaultType ()) in Parser.eatBreadcrumb p; typ From 32387e3b415f4a57cb1bd4ec9994afb61380efa4 Mon Sep 17 00:00:00 2001 From: Sehoon Kim Date: Mon, 4 Jul 2022 19:27:09 +0900 Subject: [PATCH 4/5] add error handling & fix pattern matching --- src/res_core.ml | 88 ++++++++++++++++++++++++------------------------- 1 file changed, 43 insertions(+), 45 deletions(-) diff --git a/src/res_core.ml b/src/res_core.ml index f9b0954b..a67838bc 100644 --- a/src/res_core.ml +++ b/src/res_core.ml @@ -556,14 +556,13 @@ let rec parseLident p = Parser.next p; let loc = mkLoc startPos p.prevEndPos in (ident, loc) - | token -> ( - if token = Eof then ( - Parser.err ~startPos p (Diagnostics.unexpected token p.breadcrumbs); - ("_", mkLoc startPos p.prevEndPos)) - else - match recoverLident p with - | Some () -> parseLident p - | None -> ("_", mkLoc startPos p.prevEndPos)) + | token when token = Eof -> + Parser.err ~startPos p (Diagnostics.unexpected token p.breadcrumbs); + ("_", mkLoc startPos p.prevEndPos) + | _ -> ( + match recoverLident p with + | Some () -> parseLident p + | None -> ("_", mkLoc startPos p.prevEndPos)) let parseIdent ~msg ~startPos p = match p.Parser.token with @@ -604,11 +603,10 @@ let parseHashIdent ~startPos p = in Parser.next p; (i, mkLoc startPos p.prevEndPos) - | token -> - if token = Eof then ( - Parser.err ~startPos p (Diagnostics.unexpected token p.breadcrumbs); - ("", mkLoc startPos p.prevEndPos)) - else parseIdent ~startPos ~msg:ErrorMessages.variantIdent p + | token when token = Eof -> + Parser.err ~startPos p (Diagnostics.unexpected token p.breadcrumbs); + ("", mkLoc startPos p.prevEndPos) + | _ -> parseIdent ~startPos ~msg:ErrorMessages.variantIdent p (* Ldot (Ldot (Lident "Foo", "Bar"), "baz") *) let parseValuePath p = @@ -1098,12 +1096,10 @@ let rec parsePattern ?(alias = true) ?(or_ = true) p = in Parser.next p; (i, mkLoc startPos p.prevEndPos) - | token -> - if token = Eof then ( - Parser.err ~startPos p - (Diagnostics.unexpected token p.breadcrumbs); - ("", mkLoc startPos p.prevEndPos)) - else parseIdent ~msg:ErrorMessages.variantIdent ~startPos p + | token when token = Eof -> + Parser.err ~startPos p (Diagnostics.unexpected token p.breadcrumbs); + ("", mkLoc startPos p.prevEndPos) + | _ -> parseIdent ~msg:ErrorMessages.variantIdent ~startPos p in match p.Parser.token with | Lparen -> parseVariantPatternArgs p ident startPos attrs @@ -1126,16 +1122,16 @@ let rec parsePattern ?(alias = true) ?(or_ = true) p = let extension = parseExtension p in let loc = mkLoc startPos p.prevEndPos in Ast_helper.Pat.extension ~loc ~attrs extension + | token when token = Eof -> + Parser.err p (Diagnostics.unexpected token p.breadcrumbs); + Recover.defaultPattern () | token -> ( Parser.err p (Diagnostics.unexpected token p.breadcrumbs); - if p.Parser.token = Eof then Recover.defaultPattern () - else - match - skipTokensAndMaybeRetry p - ~isStartOfGrammar:Grammar.isAtomicPatternStart - with - | None -> Recover.defaultPattern () - | Some () -> parsePattern p) + match + skipTokensAndMaybeRetry p ~isStartOfGrammar:Grammar.isAtomicPatternStart + with + | None -> Recover.defaultPattern () + | Some () -> parsePattern p) in let pat = if alias then parseAliasPattern ~attrs pat p else pat in if or_ then parseOrPattern pat p else pat @@ -1875,16 +1871,18 @@ and parseAtomicExpr p = Parser.err p (Diagnostics.lident token); Parser.next p; Recover.defaultExpr () + | token when token = Eof -> + Parser.err ~startPos:p.prevEndPos p + (Diagnostics.unexpected token p.breadcrumbs); + Recover.defaultExpr () | token -> ( let errPos = p.prevEndPos in Parser.err ~startPos:errPos p (Diagnostics.unexpected token p.breadcrumbs); - if p.Parser.token = Eof then Recover.defaultExpr () - else - match - skipTokensAndMaybeRetry p ~isStartOfGrammar:Grammar.isAtomicExprStart - with - | None -> Recover.defaultExpr () - | Some () -> parseAtomicExpr p) + match + skipTokensAndMaybeRetry p ~isStartOfGrammar:Grammar.isAtomicExprStart + with + | None -> Recover.defaultExpr () + | Some () -> parseAtomicExpr p) in Parser.eatBreadcrumb p; expr @@ -3879,19 +3877,19 @@ and parseAtomicTypExpr ~attrs p = let loc = mkLoc startPos p.prevEndPos in Ast_helper.Typ.extension ~attrs ~loc extension | Lbrace -> parseRecordOrObjectType ~attrs p + | token when token = Eof -> + Parser.err p (Diagnostics.unexpected token p.breadcrumbs); + Recover.defaultType () | token -> ( Parser.err p (Diagnostics.unexpected token p.breadcrumbs); - if p.Parser.token = Eof then Recover.defaultType () - else - match - skipTokensAndMaybeRetry p - ~isStartOfGrammar:Grammar.isAtomicTypExprStart - with - | Some () -> parseAtomicTypExpr ~attrs p - | None -> - Parser.err ~startPos:p.prevEndPos p - (Diagnostics.unexpected token p.breadcrumbs); - Recover.defaultType ()) + match + skipTokensAndMaybeRetry p ~isStartOfGrammar:Grammar.isAtomicTypExprStart + with + | Some () -> parseAtomicTypExpr ~attrs p + | None -> + Parser.err ~startPos:p.prevEndPos p + (Diagnostics.unexpected token p.breadcrumbs); + Recover.defaultType ()) in Parser.eatBreadcrumb p; typ From 12d00ebd729e9703caa2d2157093820996272f27 Mon Sep 17 00:00:00 2001 From: Sehoon Kim Date: Mon, 4 Jul 2022 21:32:51 +0900 Subject: [PATCH 5/5] fix pattern match Eof --- src/res_core.ml | 25 +++++++++++++------------ 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/src/res_core.ml b/src/res_core.ml index a67838bc..b1e0530b 100644 --- a/src/res_core.ml +++ b/src/res_core.ml @@ -556,8 +556,8 @@ let rec parseLident p = Parser.next p; let loc = mkLoc startPos p.prevEndPos in (ident, loc) - | token when token = Eof -> - Parser.err ~startPos p (Diagnostics.unexpected token p.breadcrumbs); + | Eof -> + Parser.err ~startPos p (Diagnostics.unexpected p.Parser.token p.breadcrumbs); ("_", mkLoc startPos p.prevEndPos) | _ -> ( match recoverLident p with @@ -603,8 +603,8 @@ let parseHashIdent ~startPos p = in Parser.next p; (i, mkLoc startPos p.prevEndPos) - | token when token = Eof -> - Parser.err ~startPos p (Diagnostics.unexpected token p.breadcrumbs); + | Eof -> + Parser.err ~startPos p (Diagnostics.unexpected p.token p.breadcrumbs); ("", mkLoc startPos p.prevEndPos) | _ -> parseIdent ~startPos ~msg:ErrorMessages.variantIdent p @@ -1096,8 +1096,9 @@ let rec parsePattern ?(alias = true) ?(or_ = true) p = in Parser.next p; (i, mkLoc startPos p.prevEndPos) - | token when token = Eof -> - Parser.err ~startPos p (Diagnostics.unexpected token p.breadcrumbs); + | Eof -> + Parser.err ~startPos p + (Diagnostics.unexpected p.token p.breadcrumbs); ("", mkLoc startPos p.prevEndPos) | _ -> parseIdent ~msg:ErrorMessages.variantIdent ~startPos p in @@ -1122,8 +1123,8 @@ let rec parsePattern ?(alias = true) ?(or_ = true) p = let extension = parseExtension p in let loc = mkLoc startPos p.prevEndPos in Ast_helper.Pat.extension ~loc ~attrs extension - | token when token = Eof -> - Parser.err p (Diagnostics.unexpected token p.breadcrumbs); + | Eof -> + Parser.err p (Diagnostics.unexpected p.Parser.token p.breadcrumbs); Recover.defaultPattern () | token -> ( Parser.err p (Diagnostics.unexpected token p.breadcrumbs); @@ -1871,9 +1872,9 @@ and parseAtomicExpr p = Parser.err p (Diagnostics.lident token); Parser.next p; Recover.defaultExpr () - | token when token = Eof -> + | Eof -> Parser.err ~startPos:p.prevEndPos p - (Diagnostics.unexpected token p.breadcrumbs); + (Diagnostics.unexpected p.Parser.token p.breadcrumbs); Recover.defaultExpr () | token -> ( let errPos = p.prevEndPos in @@ -3877,8 +3878,8 @@ and parseAtomicTypExpr ~attrs p = let loc = mkLoc startPos p.prevEndPos in Ast_helper.Typ.extension ~attrs ~loc extension | Lbrace -> parseRecordOrObjectType ~attrs p - | token when token = Eof -> - Parser.err p (Diagnostics.unexpected token p.breadcrumbs); + | Eof -> + Parser.err p (Diagnostics.unexpected p.Parser.token p.breadcrumbs); Recover.defaultType () | token -> ( Parser.err p (Diagnostics.unexpected token p.breadcrumbs);