Parse extended syntax

If the `(?x)` or `(?xx)` options are active in
a given scope, treat whitespace as non-semantic,
including in custom character classes. Additionally
parse end-of-line comments `# comment`.

Author: hamishknight

Sources/_MatchingEngine/Regex/AST/CustomCharClass.swift

@@ -42,6 +42,9 @@ extension AST {
       /// the contents should be interpreted literally.
       case quote(Quote)
 
+      /// Trivia such as non-semantic whitespace.
+      case trivia(Trivia)
+
       /// A binary operator applied to sets of members `abc&&def`
       case setOperation([Member], Located<SetOp>, [Member])
     }
@@ -81,6 +84,7 @@ extension CustomCC.Member {
     case .range(let r): return r
     case .atom(let a): return a
     case .quote(let q): return q
+    case .trivia(let t): return t
     case .setOperation(let lhs, let op, let rhs): return (lhs, op, rhs)
     }
   }

Sources/_MatchingEngine/Regex/AST/MatchingOptions.swift

@@ -50,6 +50,16 @@ extension AST {
       self.location = location
     }
 
+    /// If this is either the regular or extra extended syntax option.
+    public var isAnyExtended: Bool {
+      switch kind {
+      case .extended, .extraExtended:
+        return true
+      default:
+        return false
+      }
+    }
+
     public var isTextSegmentMode: Bool {
       switch kind {
       case .textSegmentGraphemeMode, .textSegmentWordMode:
@@ -93,6 +103,10 @@ extension AST {
       self.minusLoc = minusLoc
       self.removing = removing
     }
+
+    /// Whether this set of matching options first resets the options before
+    /// adding onto them.
+    public var resetsCurrentOptions: Bool { caretLoc != nil }
   }
 }
 
@@ -102,7 +116,10 @@ extension AST.MatchingOption: _ASTPrintable {
 
 extension AST.MatchingOptionSequence: _ASTPrintable {
   public var _dumpBase: String {
-    "adding: \(adding), removing: \(removing), hasCaret: \(caretLoc != nil)"
+    """
+    adding: \(adding), removing: \(removing), \
+    resetsCurrentOptions: \(resetsCurrentOptions)
+    """
   }
 }
 

Sources/_MatchingEngine/Regex/AST/Quantification.swift

@@ -17,16 +17,23 @@ extension AST {
     public let child: AST.Node
     public let location: SourceLocation
 
+    /// Any trivia intermixed between the operand and the quantifier, as well
+    /// as between the quantifier characters themselves. This can occur in
+    /// extended syntax mode where PCRE permits e.g `x * +`.
+    public let trivia: [AST.Trivia]
+
     public init(
       _ amount: Located<Amount>,
       _ kind: Located<Kind>,
       _ child: AST.Node,
-      _ r: SourceLocation
+      _ r: SourceLocation,
+      trivia: [AST.Trivia]
     ) {
       self.amount = amount
       self.kind = kind
       self.child = child
       self.location = r
+      self.trivia = trivia
     }
 
     @frozen

Sources/_MatchingEngine/Regex/Parse/LexicalAnalysis.swift

@@ -332,9 +332,15 @@ extension Source {
   ///     Quantifier -> ('*' | '+' | '?' | '{' Range '}') QuantKind?
   ///     QuantKind  -> '?' | '+'
   ///
-  mutating func lexQuantifier(context: ParsingContext) throws -> (
-    Located<Quant.Amount>, Located<Quant.Kind>
-  )? {
+  mutating func lexQuantifier(
+    context: ParsingContext
+  ) throws -> (Located<Quant.Amount>, Located<Quant.Kind>, [AST.Trivia])? {
+    var trivia: [AST.Trivia] = []
+
+    if let t = try lexNonSemanticWhitespace(context: context) {
+      trivia.append(t)
+    }
+
     let amt: Located<Quant.Amount>? = try recordLoc { src in
       if src.tryEat("*") { return .zeroOrMore }
       if src.tryEat("+") { return .oneOrMore }
@@ -350,13 +356,18 @@ extension Source {
     }
     guard let amt = amt else { return nil }
 
+    // PCRE allows non-semantic whitespace here in extended syntax mode.
+    if let t = try lexNonSemanticWhitespace(context: context) {
+      trivia.append(t)
+    }
+
     let kind: Located<Quant.Kind> = recordLoc { src in
       if src.tryEat("?") { return .reluctant  }
       if src.tryEat("+") { return .possessive }
       return .eager
     }
 
-    return (amt, kind)
+    return (amt, kind, trivia)
   }
 
   /// Try to consume a range, returning `nil` if unsuccessful.
@@ -501,6 +512,10 @@ extension Source {
   ///
   ///     ExpComment -> '/*' (!'*/' .)* '*/'
   ///
+  /// With `SyntaxOptions.endOfLineComments`
+  ///
+  ///     EndOfLineComment -> '#' .*
+  ///
   /// TODO: Swift-style nested comments, line-ending comments, etc
   ///
   mutating func lexComment(context: ParsingContext) throws -> AST.Trivia? {
@@ -511,6 +526,13 @@ extension Source {
       if context.experimentalComments, src.tryEat(sequence: "/*") {
         return try src.expectQuoted(endingWith: "*/").value
       }
+      if context.endOfLineComments, src.tryEat("#") {
+        // TODO: If we ever support multi-line regex literals, this will need
+        // to be updated to stop at a newline. Note though that PCRE specifies
+        // that the newline it matches against can be controlled by the global
+        // matching options e.g `(*CR)`, `(*ANY)`, ...
+        return src.lexUntil(\.isEmpty).value
+      }
       return nil
     }
     guard let trivia = trivia else { return nil }
@@ -519,15 +541,36 @@ extension Source {
 
   /// Try to consume non-semantic whitespace as trivia
   ///
-  ///     Whitespace -> ' '+
+  ///     Whitespace -> WhitespaceChar+
   ///
   /// Does nothing unless `SyntaxOptions.nonSemanticWhitespace` is set
   mutating func lexNonSemanticWhitespace(
     context: ParsingContext
   ) throws -> AST.Trivia? {
     guard context.ignoreWhitespace else { return nil }
+
+    func isWhitespace(_ c: Character) -> Bool {
+      // This is a list of characters that PCRE treats as whitespace when
+      // compiled with Unicode support. It is a subset of the characters with
+      // the `.isWhitespace` property. ICU appears to also follow this list.
+      // Oniguruma and .NET follow a subset of this list.
+      //
+      // FIXME: PCRE only treats space and tab characters as whitespace when
+      // inside a custom character class (and only treats whitespace as
+      // non-semantic there for the extra-extended `(?xx)` mode). If we get a
+      // strict-PCRE mode, we'll need to add a case for that.
+      switch c {
+      case " ", "\u{9}"..."\u{D}", // space, \t, \n, vertical tab, \f, \r
+           "\u{85}", "\u{200E}",   // next line, left-to-right mark
+           "\u{200F}", "\u{2028}", // right-to-left-mark, line separator
+           "\u{2029}":             // paragraph separator
+        return true
+      default:
+        return false
+      }
+    }
     let trivia: Located<String>? = recordLoc { src in
-      src.tryEatPrefix { $0 == " " }?.string
+      src.tryEatPrefix(isWhitespace)?.string
     }
     guard let trivia = trivia else { return nil }
     return AST.Trivia(trivia)
@@ -1631,7 +1674,7 @@ extension Source {
       var name: Located<String>?
       if src.tryEat(":") {
         // TODO: PCRE allows escaped delimiters or '\Q...\E' sequences in the
-        // name under PCRE2_ALT_VERBNAMES.
+        // name under PCRE2_ALT_VERBNAMES. It also allows whitespace under (?x).
         name = try src.expectQuoted(endingWith: ")", eatEnding: false)
       }
       try src.expect(")")

Sources/_MatchingEngine/Regex/Parse/Parse.swift

@@ -106,6 +106,7 @@ extension ParsingContext {
   var experimentalQuotes: Bool { syntax.contains(.experimentalQuotes) }
   var experimentalComments: Bool { syntax.contains(.experimentalComments) }
   var ignoreWhitespace: Bool { syntax.contains(.nonSemanticWhitespace) }
+  var endOfLineComments: Bool { syntax.contains(.endOfLineComments) }
 }
 
 // Diagnostics
@@ -208,12 +209,14 @@ extension Parser {
       }
       //     Quantification  -> QuantOperand Quantifier?
       if let operand = try parseQuantifierOperand() {
-        if let (amt, kind) = try source.lexQuantifier(context: context) {
+        if let (amt, kind, trivia) =
+            try source.lexQuantifier(context: context) {
           let location = loc(_start)
           guard operand.isQuantifiable else {
             throw Source.LocatedError(ParseError.notQuantifiable, location)
           }
-          result.append(.quantification(.init(amt, kind, operand, location)))
+          result.append(.quantification(
+            .init(amt, kind, operand, location, trivia: trivia)))
         } else {
           result.append(operand)
         }
@@ -270,6 +273,28 @@ extension Parser {
   ) throws -> AST.Group {
     context.recordGroup(kind.value)
 
+    // Check if we're introducing or removing extended syntax.
+    // TODO: PCRE differentiates between (?x) and (?xx) where only the latter
+    // handles non-semantic whitespace in a custom character class. Other
+    // engines such as Oniguruma, Java, and ICU do this under (?x). Therefore,
+    // treat (?x) and (?xx) as the same option here. If we ever get a strict
+    // PCRE mode, we will need to change this to handle that.
+    let currentSyntax = context.syntax
+    if case .changeMatchingOptions(let c, isIsolated: _) = kind.value {
+      if c.resetsCurrentOptions {
+        context.syntax.remove(.extendedSyntax)
+      }
+      if c.adding.contains(where: \.isAnyExtended) {
+        context.syntax.insert(.extendedSyntax)
+      }
+      if c.removing.contains(where: \.isAnyExtended) {
+        context.syntax.remove(.extendedSyntax)
+      }
+    }
+    defer {
+      context.syntax = currentSyntax
+    }
+
     let child = try parseNode()
     // An implicit scoped group has already consumed its closing paren.
     if !kind.value.hasImplicitScope {
@@ -449,6 +474,13 @@ extension Parser {
         continue
       }
 
+      // Lex non-semantic whitespace if we're allowed.
+      // TODO: ICU allows end-of-line comments in custom character classes,
+      // which we ought to support if we want to support multi-line regex.
+      if let trivia = try source.lexNonSemanticWhitespace(context: context) {
+        members.append(.trivia(trivia))
+      }
+
       guard let atom = try source.lexAtom(context: context) else { break }
 
       // Range between atoms.

Sources/_MatchingEngine/Regex/Parse/SyntaxOptions.swift

@@ -23,19 +23,27 @@ public struct SyntaxOptions: OptionSet {
   /// `'a \. b' == '/a\.b/'`
   public static var nonSemanticWhitespace: Self { Self(1 << 0) }
 
+  /// `abc # comment`
+  public static var endOfLineComments: Self { Self(1 << 1) }
+
+  /// `(?x)` `(?xx)`
+  public static var extendedSyntax: Self {
+    [.endOfLineComments, .nonSemanticWhitespace]
+  }
+
   /// `'a "." b' == '/a\Q.\Eb/'`
   ///
   /// NOTE: Currently, this means we have raw quotes.
   /// Better would be to have real Swift string delimiter parsing logic.
-  public static var experimentalQuotes: Self { Self(1 << 1) }
+  public static var experimentalQuotes: Self { Self(1 << 2) }
 
   /// `'a /* comment */ b' == '/a(?#. comment )b/'`
   ///
   /// NOTE: traditional comments are not nested. Currently, we are neither.
   /// Traditional comments can't have `)`, not even escaped in them either, we
   /// can. Traditional comments can have `*/` in them, we can't without
   /// escaping. We don't currently do escaping.
-  public static var experimentalComments: Self { Self(1 << 2) }
+  public static var experimentalComments: Self { Self(1 << 3) }
 
   /// ```
   ///   'a{n...m}' == '/a{n,m}/'
@@ -44,11 +52,11 @@ public struct SyntaxOptions: OptionSet {
   ///   'a{...m}'  == '/a{,m}/'
   ///   'a{..<m}'  == '/a{,m-1}/'
   /// ```
-  public static var experimentalRanges: Self { Self(1 << 3) }
+  public static var experimentalRanges: Self { Self(1 << 4) }
 
   /// `(name: .*)` == `(?<name>.*)`
   ///  `(_: .*)` == `(?:.*)`
-  public static var experimentalCaptures: Self { Self(1 << 4) }
+  public static var experimentalCaptures: Self { Self(1 << 5) }
 
   /*
 
@@ -59,10 +67,9 @@ public struct SyntaxOptions: OptionSet {
 
   public static var traditional: Self { Self(0) }
 
-  public static var experimental: Self { Self(~0) }
-
-  public var ignoreWhitespace: Bool {
-    contains(.nonSemanticWhitespace)
+  public static var experimental: Self {
+    // Experimental syntax enables everything except end-of-line comments.
+    Self(~0).subtracting(.endOfLineComments)
   }
 
   // TODO: Probably want to model strict-PCRE etc. options too.

Sources/_MatchingEngine/Regex/Printing/DumpAST.swift

@@ -52,6 +52,9 @@ extension _ASTPrintable {
       if $0.isTrivia { return nil }
       return $0._dump()
     }.joined(separator: ",")
+    if sub.isEmpty {
+      return "\(_dumpBase)"
+    }
     return "\(_dumpBase)(\(sub))"
   }
 }
@@ -287,7 +290,15 @@ extension AST.Quantification: _ASTPrintable {
 
 extension AST.CustomCharacterClass: _ASTNode {
   public var _dumpBase: String {
-    "customCharacterClass(\(members))"
+    // Exclude trivia for now, as we don't want it to appear when performing
+    // comparisons of dumped output in tests.
+    // TODO: We should eventually have some way of filtering out trivia for
+    // tests, so that it can appear in regular dumps.
+    let semanticMembers = members.filter {
+      if case .trivia = $0 { return false }
+      return true
+    }
+    return "customCharacterClass(\(semanticMembers))"
   }
 }
 
@@ -298,6 +309,7 @@ extension AST.CustomCharacterClass.Member: _ASTPrintable {
     case .atom(let a): return "\(a)"
     case .range(let r): return "\(r)"
     case .quote(let q): return "\(q)"
+    case .trivia(let t): return "\(t)"
     case .setOperation(let lhs, let op, let rhs):
       return "op \(lhs) \(op.value) \(rhs)"
     }

Sources/_MatchingEngine/Regex/Printing/PrintAsCanonical.swift

@@ -91,9 +91,7 @@ extension PrettyPrinter {
       output(q._canonicalBase)
 
     case let .trivia(t):
-      // TODO: We might want to output comments...
-      _ = t
-      output("")
+      output(t._canonicalBase)
 
     case let .atom(a):
       output(a._canonicalBase)
@@ -135,6 +133,8 @@ extension PrettyPrinter {
       output(a._canonicalBase)
     case .quote(let q):
       output(q._canonicalBase)
+    case .trivia(let t):
+      output(t._canonicalBase)
     case .setOperation:
       output("/* TODO: set operation \(self) */")
     }
@@ -315,3 +315,10 @@ extension AST.GlobalMatchingOption.Kind {
 extension AST.GlobalMatchingOption {
   var _canonicalBase: String { "(*\(kind._canonicalBase))"}
 }
+
+extension AST.Trivia {
+  var _canonicalBase: String {
+    // TODO: We might want to output comments...
+    ""
+  }
+}

Sources/_MatchingEngine/Regex/Printing/PrintAsPattern.swift

@@ -180,6 +180,9 @@ extension PrettyPrinter {
       }
     case .quote(let q):
       print("// TODO: quote \(q.literal._quoted) in custom character classes (should we split it?)")
+    case .trivia(let t):
+      // TODO: We might want to output comments...
+      _ = t
     case .setOperation:
       print("// TODO: Set operation: \(member)")
     }