feat(syntax/parser): add basic regexp literal support

Author: glennsl

jscomp/syntax/src/res_core.ml

@@ -1891,6 +1891,22 @@ and parse_constrained_expr_region p =
     | _ -> Some expr)
   | _ -> None
 
+and parse_regex p pattern flags =
+  let start_pos = p.Parser.start_pos in
+  Parser.next p;
+  let loc = mk_loc start_pos p.prev_end_pos in
+  let payload =
+    Parsetree.PStr
+      [
+        Ast_helper.Str.eval ~loc
+          (Ast_helper.Exp.constant ~loc
+             (Pconst_string
+                ( "/" ^ pattern ^ "/" ^ flags,
+                  if p.mode = ParseForTypeChecker then Some "js" else None )));
+      ]
+  in
+  Ast_helper.Exp.extension (Location.mknoloc "re", payload)
+
 (* Atomic expressions represent unambiguous expressions.
  * This means that regardless of the context, these expressions
  * are always interpreted correctly. *)
@@ -1960,6 +1976,18 @@ and parse_atomic_expr p =
       Parser.err ~start_pos:p.prev_end_pos p
         (Diagnostics.unexpected p.Parser.token p.breadcrumbs);
       Recover.default_expr ()
+    | Forwardslash -> (
+      Parser.next_regex_token p;
+      match p.token with
+      | Regex (pattern, flags) -> parse_regex p pattern flags
+      | _ -> Ast_helper.Exp.extension (Location.mknoloc "re", Parsetree.PStr [])
+      )
+    | ForwardslashDot -> (
+      Parser.next_regex_token p;
+      match p.token with
+      | Regex (pattern, flags) -> parse_regex p ("." ^ pattern) flags
+      | _ -> Ast_helper.Exp.extension (Location.mknoloc "re", Parsetree.PStr [])
+      )
     | token -> (
       let err_pos = p.prev_end_pos in
       Parser.err ~start_pos:err_pos p

jscomp/syntax/src/res_grammar.ml

@@ -136,7 +136,7 @@ let is_atomic_pattern_start = function
 let is_atomic_expr_start = function
   | Token.True | False | Int _ | String _ | Float _ | Codepoint _ | Backtick
   | Uident _ | Lident _ | Hash | Lparen | List | Lbracket | Lbrace | LessThan
-  | Module | Percent ->
+  | Module | Percent | Forwardslash | ForwardslashDot ->
     true
   | _ -> false
 
@@ -151,7 +151,7 @@ let is_expr_start = function
   | For | Hash | If | Int _ | Lbrace | Lbracket | LessThan | Lident _ | List
   | Lparen | Minus | MinusDot | Module | Percent | Plus | PlusDot | String _
   | Switch | True | Try | Uident _ | Underscore (* _ => doThings() *)
-  | While ->
+  | While | Forwardslash | ForwardslashDot ->
     true
   | _ -> false
 
@@ -257,10 +257,10 @@ let is_jsx_child_start = is_atomic_expr_start
 
 let is_block_expr_start = function
   | Token.Assert | At | Await | Backtick | Bang | Codepoint _ | Exception
-  | False | Float _ | For | Forwardslash | Hash | If | Int _ | Lbrace | Lbracket
-  | LessThan | Let | Lident _ | List | Lparen | Minus | MinusDot | Module | Open
-  | Percent | Plus | PlusDot | String _ | Switch | True | Try | Uident _
-  | Underscore | While ->
+  | False | Float _ | For | Forwardslash | ForwardslashDot | Hash | If | Int _
+  | Lbrace | Lbracket | LessThan | Let | Lident _ | List | Lparen | Minus
+  | MinusDot | Module | Open | Percent | Plus | PlusDot | String _ | Switch
+  | True | Try | Uident _ | Underscore | While ->
     true
   | _ -> false
 

jscomp/syntax/src/res_parser.ml

@@ -106,6 +106,13 @@ let next_template_literal_token p =
   p.start_pos <- start_pos;
   p.end_pos <- end_pos
 
+let next_regex_token p =
+  let start_pos, end_pos, token = Scanner.scan_regex p.scanner in
+  p.token <- token;
+  p.prev_end_pos <- p.end_pos;
+  p.start_pos <- start_pos;
+  p.end_pos <- end_pos
+
 let check_progress ~prev_end_pos ~result p =
   if p.end_pos == prev_end_pos then None else Some result
 

jscomp/syntax/src/res_parser.mli

@@ -31,6 +31,7 @@ val optional : t -> Token.t -> bool
 val next : ?prev_end_pos:Lexing.position -> t -> unit
 val next_unsafe : t -> unit (* Does not assert on Eof, makes no progress *)
 val next_template_literal_token : t -> unit
+val next_regex_token : t -> unit
 val lookahead : t -> (t -> 'a) -> 'a
 val err :
   ?start_pos:Lexing.position ->

jscomp/syntax/src/res_scanner.ml

@@ -537,6 +537,64 @@ let scan_escape scanner =
   (* TODO: do we know it's \' ? *)
   Token.Codepoint {c = codepoint; original = contents}
 
+let scan_regex scanner =
+  let start_pos = position scanner in
+  let buf = Buffer.create 0 in
+  let first_char_offset = scanner.offset in
+  let last_offset_in_buf = ref first_char_offset in
+
+  let bring_buf_up_to_date ~start_offset =
+    let str_up_to_now =
+      (String.sub scanner.src !last_offset_in_buf
+         (start_offset - !last_offset_in_buf) [@doesNotRaise])
+    in
+    Buffer.add_string buf str_up_to_now;
+    last_offset_in_buf := start_offset
+  in
+
+  let result ~first_char_offset ~last_char_offset =
+    if Buffer.length buf = 0 then
+      (String.sub [@doesNotRaise]) scanner.src first_char_offset
+        (last_char_offset - first_char_offset)
+    else (
+      bring_buf_up_to_date ~start_offset:last_char_offset;
+      Buffer.contents buf)
+  in
+  let rec scan () =
+    match scanner.ch with
+    | '/' ->
+      let last_char_offset = scanner.offset in
+      next scanner;
+      let pattern = result ~first_char_offset ~last_char_offset in
+      let flags =
+        let flags_buf = Buffer.create 0 in
+        let rec scan_flags () =
+          match scanner.ch with
+          | 'd' | 'g' | 'i' | 'm' | 's' | 'u' | 'v' | 'y' ->
+            Buffer.add_char flags_buf scanner.ch;
+            next scanner;
+            scan_flags ()
+          | _ -> Buffer.contents flags_buf
+        in
+        scan_flags ()
+      in
+      (pattern, flags)
+    | ch when ch == '\n' || ch == hacky_eof_char ->
+      let end_pos = position scanner in
+      scanner.err ~start_pos ~end_pos (Diagnostics.message "unterminated regex");
+      ("", "")
+    | '\\' ->
+      next scanner;
+      next scanner;
+      scan ()
+    | _ ->
+      next scanner;
+      scan ()
+  in
+  let pattern, flags = scan () in
+  let end_pos = position scanner in
+  (start_pos, end_pos, Token.Regex (pattern, flags))
+
 let scan_single_line_comment scanner =
   let start_off = scanner.offset in
   let start_pos = position scanner in

jscomp/syntax/src/res_scanner.mli

@@ -34,3 +34,5 @@ val reconsider_less_than : t -> Res_token.t
 
 val scan_template_literal_token :
   t -> Lexing.position * Lexing.position * Res_token.t
+
+val scan_regex : t -> Lexing.position * Lexing.position * Res_token.t

jscomp/syntax/src/res_token.ml

@@ -39,6 +39,7 @@ type t =
   | Backslash [@live]
   | Forwardslash
   | ForwardslashDot
+  | Regex of string * string
   | Asterisk
   | AsteriskDot
   | Exponentiation
@@ -153,6 +154,7 @@ let to_string = function
   | PlusPlus -> "++"
   | PlusEqual -> "+="
   | Backslash -> "\\"
+  | Regex (pattern, flags) -> "regex: /" ^ pattern ^ "/" ^ flags
   | Forwardslash -> "/"
   | ForwardslashDot -> "/."
   | Exception -> "exception"