Move SyntaxOptions from Source to ParserContext

And plumb through `context` parameters for more
lexing methods. This is necessary as `(?x)` will
be able to affect the syntax options set during
parsing.

Author: hamishknight

Sources/_MatchingEngine/Regex/Parse/LexicalAnalysis.swift

@@ -332,7 +332,7 @@ extension Source {
   ///     Quantifier -> ('*' | '+' | '?' | '{' Range '}') QuantKind?
   ///     QuantKind  -> '?' | '+'
   ///
-  mutating func lexQuantifier() throws -> (
+  mutating func lexQuantifier(context: ParsingContext) throws -> (
     Located<Quant.Amount>, Located<Quant.Kind>
   )? {
     let amt: Located<Quant.Amount>? = try recordLoc { src in
@@ -341,7 +341,9 @@ extension Source {
       if src.tryEat("?") { return .zeroOrOne }
 
       return try src.tryEating { src in
-        guard src.tryEat("{"), let range = try src.lexRange(), src.tryEat("}")
+        guard src.tryEat("{"),
+              let range = try src.lexRange(context: context),
+              src.tryEat("}")
         else { return nil }
         return range.value
       }
@@ -363,7 +365,7 @@ extension Source {
   ///                  | ExpRange
   ///     ExpRange    -> '..<' <Int> | '...' <Int>
   ///                  | <Int> '..<' <Int> | <Int> '...' <Int>?
-  mutating func lexRange() throws -> Located<Quant.Amount>? {
+  mutating func lexRange(context: ParsingContext) throws -> Located<Quant.Amount>? {
     try recordLoc { src in
       try src.tryEating { src in
         let lowerOpt = try src.lexNumber()
@@ -375,7 +377,7 @@ extension Source {
         let closedRange: Bool?
         if src.tryEat(",") {
           closedRange = true
-        } else if src.experimentalRanges && src.tryEat(".") {
+        } else if context.experimentalRanges && src.tryEat(".") {
           try src.expect(".")
           if src.tryEat(".") {
             closedRange = true
@@ -477,12 +479,12 @@ extension Source {
   ///
   /// TODO: Need to support some escapes
   ///
-  mutating func lexQuote() throws -> AST.Quote? {
+  mutating func lexQuote(context: ParsingContext) throws -> AST.Quote? {
     let str = try recordLoc { src -> String? in
       if src.tryEat(sequence: #"\Q"#) {
         return try src.expectQuoted(endingWith: #"\E"#).value
       }
-      if src.experimentalQuotes, src.tryEat("\"") {
+      if context.experimentalQuotes, src.tryEat("\"") {
         return try src.expectQuoted(endingWith: "\"", ignoreEscaped: true).value
       }
       return nil
@@ -501,12 +503,12 @@ extension Source {
   ///
   /// TODO: Swift-style nested comments, line-ending comments, etc
   ///
-  mutating func lexComment() throws -> AST.Trivia? {
+  mutating func lexComment(context: ParsingContext) throws -> AST.Trivia? {
     let trivia: Located<String>? = try recordLoc { src in
       if src.tryEat(sequence: "(?#") {
         return try src.expectQuoted(endingWith: ")").value
       }
-      if src.experimentalComments, src.tryEat(sequence: "/*") {
+      if context.experimentalComments, src.tryEat(sequence: "/*") {
         return try src.expectQuoted(endingWith: "*/").value
       }
       return nil
@@ -517,16 +519,34 @@ extension Source {
 
   /// Try to consume non-semantic whitespace as trivia
   ///
+  ///     Whitespace -> ' '+
+  ///
   /// Does nothing unless `SyntaxOptions.nonSemanticWhitespace` is set
-  mutating func lexNonSemanticWhitespace() throws -> AST.Trivia? {
-    guard syntax.ignoreWhitespace else { return nil }
+  mutating func lexNonSemanticWhitespace(
+    context: ParsingContext
+  ) throws -> AST.Trivia? {
+    guard context.ignoreWhitespace else { return nil }
     let trivia: Located<String>? = recordLoc { src in
       src.tryEatPrefix { $0 == " " }?.string
     }
     guard let trivia = trivia else { return nil }
     return AST.Trivia(trivia)
   }
 
+  /// Try to consume trivia.
+  ///
+  ///     Trivia -> Comment | Whitespace
+  ///
+  mutating func lexTrivia(context: ParsingContext) throws -> AST.Trivia? {
+    if let comment = try lexComment(context: context) {
+      return comment
+    }
+    if let whitespace = try lexNonSemanticWhitespace(context: context) {
+      return whitespace
+    }
+    return nil
+  }
+
   /// Try to lex a matching option.
   ///
   ///     MatchingOption -> 'i' | 'J' | 'm' | 'n' | 's' | 'U' | 'x' | 'xx' | 'w'
@@ -752,6 +772,7 @@ extension Source {
   /// comments, like quotes, cannot be quantified.
   ///
   mutating func lexGroupStart(
+    context: ParsingContext
   ) throws -> Located<AST.Group.Kind>? {
     try recordLoc { src in
       try src.tryEating { src in
@@ -816,7 +837,7 @@ extension Source {
         }
 
         // (_:)
-        if src.experimentalCaptures && src.tryEat(sequence: "_:") {
+        if context.experimentalCaptures && src.tryEat(sequence: "_:") {
           return .nonCapture
         }
         // TODO: (name:)
@@ -951,9 +972,12 @@ extension Source {
   ///
   ///     GroupConditionalStart -> '(?' GroupStart
   ///
-  mutating func lexGroupConditionalStart() throws -> Located<AST.Group.Kind>? {
+  mutating func lexGroupConditionalStart(
+    context: ParsingContext
+  ) throws -> Located<AST.Group.Kind>? {
     try tryEating { src in
-      guard src.tryEat(sequence: "(?"), let group = try src.lexGroupStart()
+      guard src.tryEat(sequence: "(?"),
+            let group = try src.lexGroupStart(context: context)
       else { return nil }
 
       // Implicitly scoped groups are not supported here.

Sources/_MatchingEngine/Regex/Parse/Parse.swift

@@ -53,14 +53,17 @@ Lexical analysis provides the following:
 
 struct ParsingContext {
   /// Whether we're currently parsing in a custom character class.
-  var isInCustomCharacterClass = false
+  fileprivate(set) var isInCustomCharacterClass = false
 
   /// Tracks the number of group openings we've seen, to disambiguate the '\n'
   /// syntax as a backreference or an octal sequence.
-  fileprivate var priorGroupCount = 0
+  private var priorGroupCount = 0
 
   /// A set of used group names.
-  fileprivate var usedGroupNames = Set<String>()
+  private var usedGroupNames = Set<String>()
+
+  /// The syntax options currently set.
+  fileprivate(set) var syntax: SyntaxOptions
 
   fileprivate mutating func recordGroup(_ g: AST.Group.Kind) {
     // TODO: Needs to track group number resets (?|...).
@@ -70,8 +73,9 @@ struct ParsingContext {
     }
   }
 
-  private init() {}
-  static var none: ParsingContext { .init() }
+  init(syntax: SyntaxOptions) {
+    self.syntax = syntax
+  }
 
   /// Check whether a given reference refers to a prior group.
   func isPriorGroupRef(_ ref: AST.Reference.Kind) -> Bool {
@@ -88,13 +92,22 @@ struct ParsingContext {
 
 private struct Parser {
   var source: Source
-  var context: ParsingContext = .none
+  var context: ParsingContext
 
-  init(_ source: Source) {
+  init(_ source: Source, syntax: SyntaxOptions) {
     self.source = source
+    self.context = ParsingContext(syntax: syntax)
   }
 }
 
+extension ParsingContext {
+  var experimentalRanges: Bool { syntax.contains(.experimentalRanges) }
+  var experimentalCaptures: Bool { syntax.contains(.experimentalCaptures) }
+  var experimentalQuotes: Bool { syntax.contains(.experimentalQuotes) }
+  var experimentalComments: Bool { syntax.contains(.experimentalComments) }
+  var ignoreWhitespace: Bool { syntax.contains(.nonSemanticWhitespace) }
+}
+
 // Diagnostics
 extension Parser {
   mutating func report(
@@ -182,24 +195,20 @@ extension Parser {
       // TODO: refactor loop body into function
       let _start = source.currentPosition
 
-      //     Trivia -> `lexComment` | `lexNonSemanticWhitespace`
-      if let triv = try source.lexComment() {
-        result.append(.trivia(triv))
-        continue
-      }
-      if let triv = try source.lexNonSemanticWhitespace() {
+      //     Trivia -> `lexTrivia`
+      if let triv = try source.lexTrivia(context: context) {
         result.append(.trivia(triv))
         continue
       }
 
       //     Quote      -> `lexQuote`
-      if let quote = try source.lexQuote() {
+      if let quote = try source.lexQuote(context: context) {
         result.append(.quote(quote))
         continue
       }
       //     Quantification  -> QuantOperand Quantifier?
       if let operand = try parseQuantifierOperand() {
-        if let (amt, kind) = try source.lexQuantifier() {
+        if let (amt, kind) = try source.lexQuantifier(context: context) {
           let location = loc(_start)
           guard operand.isQuantifiable else {
             throw Source.LocatedError(ParseError.notQuantifiable, location)
@@ -333,7 +342,7 @@ extension Parser {
     if let cond = try source.lexKnownConditionalStart(context: context) {
       return try parseConditionalBranches(start: _start, cond)
     }
-    if let kind = try source.lexGroupConditionalStart() {
+    if let kind = try source.lexGroupConditionalStart(context: context) {
       let groupStart = kind.location.start
       let group = try parseGroupBody(start: groupStart, kind)
       return try parseConditionalBranches(
@@ -346,7 +355,7 @@ extension Parser {
     }
 
     // Check if we have the start of a group '('.
-    if let kind = try source.lexGroupStart() {
+    if let kind = try source.lexGroupStart(context: context) {
       return .group(try parseGroupBody(start: _start, kind))
     }
 
@@ -435,7 +444,7 @@ extension Parser {
       }
 
       // Quoted sequence.
-      if let quote = try source.lexQuote() {
+      if let quote = try source.lexQuote(context: context) {
         members.append(.quote(quote))
         continue
       }
@@ -463,8 +472,8 @@ public func parse<S: StringProtocol>(
   _ regex: S, _ syntax: SyntaxOptions
 ) throws -> AST where S.SubSequence == Substring
 {
-  let source = Source(String(regex), syntax)
-  var parser = Parser(source)
+  let source = Source(String(regex))
+  var parser = Parser(source, syntax: syntax)
   return try parser.parse()
 }
 

Sources/_MatchingEngine/Regex/Parse/Source.swift

@@ -17,15 +17,13 @@
 public struct Source {
   var input: Input
   var bounds: Range<Input.Index>
-  var syntax: SyntaxOptions
 
   // TODO: source should hold outer collection and range, at least
   // for error reporting if nothing else
 
-  init(_ str: Input, _ syntax: SyntaxOptions) {
+  init(_ str: Input) {
     self.input = str
     self.bounds = str.startIndex ..< str.endIndex
-    self.syntax = syntax
   }
 
   subscript(_ range: Range<Input.Index>) -> Input.SubSequence { input[range] }
@@ -43,18 +41,6 @@ extension Source {
   public typealias Position = String.Index
 }
 
-// MARK: - Syntax
-
-extension Source {
-  var experimentalRanges: Bool { syntax.contains(.experimentalRanges) }
-  var experimentalCaptures: Bool { syntax.contains(.experimentalCaptures) }
-  var experimentalQuotes: Bool { syntax.contains(.experimentalQuotes) }
-  var experimentalComments: Bool { syntax.contains(.experimentalComments) }
-  var nonSemanticWhitespace: Bool {
-    syntax.contains(.nonSemanticWhitespace)
-  }
-}
-
 // MARK: - Source as a peekable consumer
 
 extension Source {

Tests/RegexTests/LexTests.swift

@@ -20,7 +20,7 @@ func diagnose(
   _ syntax: SyntaxOptions = .traditional,
   _ f: (inout Source) throws -> ()
 ) {
-  var src = Source(input, syntax)
+  var src = Source(input)
   do {
     try f(&src)
     XCTFail("""