Add options support to the compiler

Sources/_MatchingEngine/Regex/AST/MatchingOptions.swift

@@ -12,7 +12,7 @@
 extension AST {
   /// An option written in source that changes matching semantics.
   public struct MatchingOption: Hashable {
-    public enum Kind {
+    public enum Kind: Int {
       // PCRE options
       case caseInsensitive          // i
       case allowDuplicateGroupNames // J
@@ -36,6 +36,11 @@ extension AST {
       // be unset, only flipped between)
       case textSegmentGraphemeMode  // y{g}
       case textSegmentWordMode      // y{w}
+
+      // Swift semantic matching level
+      case graphemeClusterSemantics // X
+      case unicodeScalarSemantics   // u
+      case byteSemantics            // b
     }
     public var kind: Kind
     public var location: SourceLocation
@@ -53,6 +58,15 @@ extension AST {
         return false
       }
     }
+
+    public var isSemanticMatchingLevel: Bool {
+      switch kind {
+      case .graphemeClusterSemantics, .unicodeScalarSemantics, .byteSemantics:
+        return true
+      default:
+        return false
+      }
+    }
   }
 
   /// A sequence of matching options written in source.
@@ -79,6 +93,74 @@ extension AST {
       self.minusLoc = minusLoc
       self.removing = removing
     }
+
+    public func options(merging optionSet: MatchingOptionSet = []) -> MatchingOptionSet {
+      var result = optionSet
+      for opt in adding {
+        if opt.isSemanticMatchingLevel {
+          result.remove(.semanticMatchingLevels)
+        }
+        if opt.isTextSegmentMode {
+          result.remove(.textSegmentOptions)
+        }
+
+        result.insert(.init(opt.kind))
+      }
+      for opt in removing {
+        result.remove(.init(opt.kind))
+      }
+      return result
+    }
+  }
+
+  /// A set of matching options.
+  public struct MatchingOptionSet: OptionSet {
+    public var rawValue: UInt32
+
+    public init(rawValue: UInt32) {
+      self.rawValue = rawValue
+    }
+
+    public init(_ kind: AST.MatchingOption.Kind) {
+      self.rawValue = 1 << kind.rawValue
+    }
+
+    // PCRE options
+    public static var caseInsensitive: Self { .init(.caseInsensitive) }
+    public static var allowDuplicateGroupNames: Self { .init(.allowDuplicateGroupNames) }
+    public static var multiline: Self { .init(.multiline) }
+    public static var noAutoCapture: Self { .init(.noAutoCapture) }
+    public static var singleLine: Self { .init(.singleLine) }
+    public static var reluctantByDefault: Self { .init(.reluctantByDefault) }
+    public static var extended: Self { .init(.extended) }
+    public static var extraExtended: Self { .init(.extraExtended) }
+
+    // ICU options
+    public static var unicodeWordBoundaries: Self { .init(.unicodeWordBoundaries) }
+
+    // Oniguruma options
+    public static var asciiOnlyDigit: Self { .init(.asciiOnlyDigit) }
+    public static var asciiOnlyPOSIXProps: Self { .init(.asciiOnlyPOSIXProps) }
+    public static var asciiOnlySpace: Self { .init(.asciiOnlySpace) }
+    public static var asciiOnlyWord: Self { .init(.asciiOnlyWord) }
+
+    // Oniguruma text segment options (these are mutually exclusive and cannot
+    // be unset, only flipped between)
+    public static var textSegmentGraphemeMode: Self { .init(.textSegmentGraphemeMode) }
+    public static var textSegmentWordMode: Self { .init(.textSegmentWordMode) }
+
+    public static var textSegmentOptions: Self {
+      [.textSegmentGraphemeMode, .textSegmentWordMode]
+    }
+
+    // Swift semantic matching level
+    public static var graphemeClusterSemantics: Self { .init(.graphemeClusterSemantics) }
+    public static var unicodeScalarSemantics: Self { .init(.unicodeScalarSemantics) }
+    public static var byteSemantics: Self { .init(.byteSemantics) }
+
+    public static var semanticMatchingLevels: Self {
+      [.graphemeClusterSemantics, .unicodeScalarSemantics, .byteSemantics]
+    }
   }
 }
 

Sources/_MatchingEngine/Regex/Parse/LexicalAnalysis.swift

@@ -550,6 +550,11 @@ extension Source {
         try src.expect("}")
         return opt
 
+      // Swift semantic level options
+      case "X": return advanceAndReturn(.graphemeClusterSemantics)
+      case "u": return advanceAndReturn(.unicodeScalarSemantics)
+      case "b": return advanceAndReturn(.byteSemantics)
+
       default:
         return nil
       }

Sources/_StringProcessing/CharacterClass.swift

@@ -338,8 +338,6 @@ extension AST.Atom {
     switch kind {
     case let .escaped(b): return b.characterClass
 
-    case .any: return .any
-
     case .property:
       // TODO: Would our model type for character classes include
       // this? Or does grapheme-semantic mode complicate that?

Sources/_StringProcessing/Compiler.swift

@@ -18,18 +18,22 @@ struct RegexProgram {
 
 class Compiler {
   let ast: AST
-  let matchLevel: CharacterClass.MatchLevel
-  let options: REOptions
+  private var optionStack: MatchingOptionSetStack
   private var builder = RegexProgram.Program.Builder()
 
+  private var currentOptions: AST.MatchingOptionSet {
+    guard let top = optionStack.top else {
+      fatalError("Unbalanced matching options removal")
+    }
+    return top
+  }
+
   init(
     ast: AST,
-    matchLevel: CharacterClass.MatchLevel = .graphemeCluster,
-    options: REOptions = []
+    options: AST.MatchingOptionSet = [.init(.graphemeClusterSemantics)]
   ) {
     self.ast = ast
-    self.matchLevel = matchLevel
-    self.options = options
+    self.optionStack = MatchingOptionSetStack(options)
   }
 
   __consuming func emit() throws -> RegexProgram {
@@ -42,11 +46,6 @@ class Compiler {
   func emit(_ node: AST) throws {
 
     switch node {
-    // Any: .
-    //     consume 1
-    case .atom(let a) where a.kind == .any && matchLevel == .graphemeCluster:
-      builder.buildAdvance(1)
-
     // Single characters we just match
     case .atom(let a) where a.singleCharacter != nil :
       builder.buildMatch(a.singleCharacter!)
@@ -91,6 +90,9 @@ class Compiler {
       break
 
     case .group(let g):
+      optionStack.push(currentOptions)
+      defer { optionStack.pop() }
+
       if let lookaround = g.lookaroundKind {
         try emitLookaround(lookaround, g.child)
         return
@@ -107,6 +109,18 @@ class Compiler {
         try emit(g.child)
         builder.buildEndCapture(cap)
 
+      case .changeMatchingOptions(let optionSequence, isIsolated: let isIsolated):
+        let updated = optionSequence.options(merging: currentOptions)
+
+        if isIsolated {
+          optionStack.replaceTop(updated)
+          try emit(g.child)
+        } else {
+          optionStack.push(updated)
+          try emit(g.child)
+          optionStack.pop()
+        }
+
       default:
         // FIXME: Other kinds...
         try emit(g.child)
@@ -118,8 +132,8 @@ class Compiler {
     // For now, we model sets and atoms as consumers.
     // This lets us rapidly expand support, and we can better
     // design the actual instruction set with real examples
-    case _ where try node.generateConsumer(matchLevel) != nil:
-      try builder.buildConsume(by: node.generateConsumer(matchLevel)!)
+    case _ where try node.generateConsumer(currentOptions) != nil:
+      try builder.buildConsume(by: node.generateConsumer(currentOptions)!)
 
     case .quote(let q):
       // We stick quoted content into read-only constant strings
@@ -474,6 +488,47 @@ class Compiler {
   }
 }
 
+/// A stack of `MatchingOptionSet`s.
+fileprivate struct MatchingOptionSetStack {
+  internal var stack: [AST.MatchingOptionSet]
+
+  init(_ initial: AST.MatchingOptionSet) {
+    self.stack = [initial]
+  }
+
+  var top: AST.MatchingOptionSet? { stack.last }
+
+  mutating func push(_ set: AST.MatchingOptionSet) {
+    stack.append(set)
+  }
+
+  mutating func replaceTop(_ set: AST.MatchingOptionSet) {
+    stack.removeLast()
+    stack.append(set)
+  }
+
+  @discardableResult
+  mutating func pop() -> AST.MatchingOptionSet {
+    stack.removeLast()
+  }
+}
+
+// Deprecated matchLevel-based initializer
+extension Compiler {
+  @available(*, deprecated)
+  convenience init(
+    ast: AST,
+    matchLevel: CharacterClass.MatchLevel,
+    options: REOptions = []
+  ) {
+    if matchLevel == .graphemeCluster {
+      self.init(ast: ast, options: .init(.graphemeClusterSemantics))
+    } else {
+      self.init(ast: ast, options: .init(.unicodeScalarSemantics))
+    }
+  }
+}
+
 public func _compileRegex(
   _ regex: String, _ syntax: SyntaxOptions = .traditional
 ) throws -> Executor {

Sources/_StringProcessing/ConsumerInterface.swift

@@ -28,12 +28,8 @@ func unsupported(
   file: StaticString = #file,
   line: UInt = #line
 ) -> Unsupported {
-  // TODO: how do we not have a public init for this?
-  let fStr = file.withUTF8Buffer {
-    String(decoding: $0, as: UTF8.self)
-  }
   return Unsupported(
-    message: s, file: fStr, line: Int(line))
+    message: s, file: String(describing: file), line: Int(line))
 }
 
 extension AST {
@@ -42,8 +38,7 @@ extension AST {
   /// A consumer is a Swift closure that matches against
   /// the front of an input range
   func generateConsumer(
-    // TODO: Better option modeling
-    _ opts: CharacterClass.MatchLevel
+    _ opts: AST.MatchingOptionSet
   ) throws -> Program<String>.ConsumeFunction? {
     switch self {
     case .atom(let a):
@@ -77,10 +72,13 @@ extension AST.Atom {
   }
 
   func generateConsumer(
-    _ opts: CharacterClass.MatchLevel
+    _ opts: AST.MatchingOptionSet
   ) throws -> Program<String>.ConsumeFunction? {
     // TODO: Wean ourselves off of this type...
-    if let cc = self.characterClass?.withMatchLevel(opts) {
+    let matchLevel: CharacterClass.MatchLevel = opts.contains(.unicodeScalarSemantics)
+      ? .unicodeScalar
+      : .graphemeCluster
+    if let cc = self.characterClass?.withMatchLevel(matchLevel) {
       return { input, bounds in
         // FIXME: should we worry about out of bounds?
         cc.matches(in: input, at: bounds.lowerBound)
@@ -109,9 +107,20 @@ extension AST.Atom {
         // TODO: alias? casing?
         $0.name == name || $0.nameAlias == name
       }
+
+    case .any:
+      return { input, bounds in
+        let curIndex = bounds.lowerBound
+        if !opts.contains(.singleLine) && input[curIndex].isNewline {
+          return nil
+        }
+        return opts.contains(.graphemeClusterSemantics)
+          ? input.index(after: curIndex)
+          : input.unicodeScalars.index(after: curIndex)
+      }
 
     case .escaped, .keyboardControl, .keyboardMeta, .keyboardMetaControl,
-        .any, .startOfLine, .endOfLine,
+        .startOfLine, .endOfLine,
         .backreference, .subpattern, .condition:
       // FIXME: implement
       return nil
@@ -121,7 +130,7 @@ extension AST.Atom {
 
 extension AST.CustomCharacterClass.Member {
   func generateConsumer(
-    _ opts: CharacterClass.MatchLevel
+    _ opts: AST.MatchingOptionSet
   ) throws -> Program<String>.ConsumeFunction {
     switch self {
     case .custom(let ccc):
@@ -212,7 +221,7 @@ extension AST.CustomCharacterClass.Member {
 
 extension AST.CustomCharacterClass {
   func generateConsumer(
-    _ opts: CharacterClass.MatchLevel
+    _ opts: AST.MatchingOptionSet
   ) throws -> Program<String>.ConsumeFunction {
     // NOTE: Easy way to implement, obviously not performant
     let consumers = try members.map {
@@ -265,7 +274,7 @@ private func consumeScalar(
 
 extension AST.Atom.CharacterProperty {
   func generateConsumer(
-    _ opts: CharacterClass.MatchLevel
+    _ opts: AST.MatchingOptionSet
   ) throws -> Program<String>.ConsumeFunction {
     // Handle inversion for us, albeit not efficiently
     func invert(
@@ -335,7 +344,7 @@ extension AST.Atom.CharacterProperty {
 extension Unicode.BinaryProperty {
   // FIXME: Semantic level, vet for precise defs
   func generateConsumer(
-    _ opts: CharacterClass.MatchLevel
+    _ opts: AST.MatchingOptionSet
   ) throws -> Program<String>.ConsumeFunction {
     switch self {
 
@@ -499,7 +508,7 @@ extension Unicode.BinaryProperty {
 extension Unicode.POSIXProperty {
   // FIXME: Semantic level, vet for precise defs
   func generateConsumer(
-    _ opts: CharacterClass.MatchLevel
+    _ opts: AST.MatchingOptionSet
   ) -> Program<String>.ConsumeFunction {
     // FIXME: semantic levels, modes, etc
     switch self {
@@ -545,7 +554,7 @@ extension Unicode.POSIXProperty {
 extension Unicode.ExtendedGeneralCategory {
   // FIXME: Semantic level
   func generateConsumer(
-    _ opts: CharacterClass.MatchLevel
+    _ opts: AST.MatchingOptionSet
   ) throws -> Program<String>.ConsumeFunction {
     switch self {
     case .letter: