Do not parse past Eof

Parsing past Eof can lead to infinite loops, as it violates the assumptions of the termination checker.
See: #540

This PR makes `Parser.next` assert false when called on Eof. This should not happen as one should check the token before calling `Parser.next`.

The one exception is during lookahead, for which we provide a `nextUnsafe` function which does not make progress.

Author: cristianoc

src/res_core.ml

@@ -259,7 +259,7 @@ let isEs6ArrowExpression ~inTernary p =
            * *)
           false
         | _ ->
-          Parser.next state;
+          Parser.nextUnsafe state;
           (* error recovery, peek at the next token,
            * (elements, providerId] => {
            *  in the example above, we have an unbalanced ] here

src/res_parser.ml

@@ -49,6 +49,7 @@ let endRegion p =
 * in the parser's state. Every comment contains the end position of its
 * previous token to facilite comment interleaving *)
 let rec next ?prevEndPos p =
+ if p.token = Eof then assert false;
  let prevEndPos = match prevEndPos with Some pos -> pos | None -> p.endPos in
  let (startPos, endPos, token) = Scanner.scan p.scanner in
  match token with
@@ -65,6 +66,9 @@ let rec next ?prevEndPos p =
    p.startPos <- startPos;
    p.endPos <- endPos
 
+let nextUnsafe p =
+  if p.token <> Eof then next p
+
 let nextTemplateLiteralToken p =
   let (startPos, endPos, token) = Scanner.scanTemplateLiteralToken p.scanner in
   p.token <- token;
@@ -82,7 +86,7 @@ let make ?(mode=ParseForTypeChecker) src filename =
   let parserState = {
     mode;
     scanner;
-    token = Token.Eof;
+    token = Token.Semicolon;
     startPos = Lexing.dummy_pos;
     prevEndPos = Lexing.dummy_pos;
     endPos = Lexing.dummy_pos;

src/res_parser.mli

@@ -28,6 +28,7 @@ val make: ?mode:mode -> string -> string -> t
 val expect: ?grammar:Grammar.t -> Token.t -> t -> unit
 val optional: t -> Token.t -> bool
 val next: ?prevEndPos:Lexing.position -> t -> unit
+val nextUnsafe: t -> unit (* Does not assert on Eof, makes no progress *)
 val nextTemplateLiteralToken: t -> unit
 val lookahead: t -> (t -> 'a) -> 'a
 val err: