Regex<Captures> -> Regex<Match>

Changes `Regex` and result builder prototype to use `Match` as the generic parameter to make it consistent with the [Strongly Typed Regex Captures](https://forums.swift.org/t/pitch-strongly-typed-regex-captures/53391) pitch.

Introduces `Tuple<n>` structs in order to be able to express constraints on capture types (i.e. `Match` dropped first) while being able to filter out empty captures in concatenation. `Tuple<n>` is also needed to implement a prototype of the [proposed matching semantics](swiftlang#64).

As coercion into `Tuple<n>` can no longer use runtime magic like native tuples do, we incorporate child capture type information into RECode's `captureNil` and `captureArray` instructions so that we will always get a concrete type when forming a nil or an empty array capture. The resulting existential tuple capture can then be opened and bitcast to a `Tuple<n>`.

Author: rxwei

Sources/Exercises/Participants/RegexParticipant.swift

@@ -33,11 +33,11 @@ struct RegexLiteralParticipant: Participant {
 }
 
 private func extractFromCaptures(
-  lower: Substring, upper: Substring?, prop: Substring
+  _ match: Tuple4<Substring, Substring, Substring?, Substring>
 ) -> GraphemeBreakEntry? {
-  guard let lowerScalar = Unicode.Scalar(hex: lower),
-        let upperScalar = upper.map(Unicode.Scalar.init(hex:)) ?? lowerScalar,
-        let property = Unicode.GraphemeBreakProperty(prop)
+  guard let lowerScalar = Unicode.Scalar(hex: match._1),
+        let upperScalar = match._2.map(Unicode.Scalar.init(hex:)) ?? lowerScalar,
+        let property = Unicode.GraphemeBreakProperty(match._3)
   else {
     return nil
   }
@@ -48,8 +48,8 @@ private func extractFromCaptures(
 private func graphemeBreakPropertyData<RP: RegexProtocol>(
   forLine line: String,
   using regex: RP
-) -> GraphemeBreakEntry? where RP.Capture == (Substring, Substring?, Substring) {
-  line.match(regex).map(\.captures).flatMap(extractFromCaptures)
+) -> GraphemeBreakEntry? where RP.Match == Tuple4<Substring, Substring, Substring?, Substring> {
+  line.match(regex).map(\.match).flatMap(extractFromCaptures)
 }
 
 private func graphemeBreakPropertyData(
@@ -75,5 +75,5 @@ private func graphemeBreakPropertyDataLiteral(
   return graphemeBreakPropertyData(
     forLine: line,
     using: r(#"([0-9A-F]+)(?:\.\.([0-9A-F]+))?\s+;\s+(\w+).*"#,
-             capturing: (Substring, Substring?, Substring).self))
+             matching: Tuple4<Substring, Substring, Substring?, Substring>.self))
 }

Sources/VariadicsGenerator/VariadicsGenerator.swift

@@ -1,11 +1,11 @@
-// swift run VariadicsGenerator --max-arity 7 > Sources/RegexDSL/Concatenation.swift
+// swift run VariadicsGenerator --max-arity 7 > Sources/_StringProcessing/RegexDSL/Concatenation.swift
 
 import ArgumentParser
 
 struct Permutation {
   let arity: Int
   // 1 -> no extra constraint
-  // 0 -> where T.Capture: NoCaptureProtocol
+  // 0 -> where T.Match: NoCaptureProtocol
   let bits: Int64
 
   func isCaptureless(at index: Int) -> Bool {
@@ -77,8 +77,6 @@ func outputForEach<C: Collection>(
     if let lt = lineTerminator {
       let indent = needsSep ? "      " : "    "
       output("\(lt)\n\(indent)")
-    } else if needsSep {
-      output(" ")
     }
   }
 }
@@ -87,12 +85,13 @@ typealias Counter = Int64
 let patternProtocolName = "RegexProtocol"
 let concatenationStructTypeBaseName = "Concatenate"
 let capturingGroupTypeBaseName = "CapturingGroup"
+let matchAssociatedTypeName = "Match"
 let captureAssociatedTypeName = "Capture"
 let patternBuilderTypeName = "RegexBuilder"
 let patternProtocolRequirementName = "regex"
 let PatternTypeBaseName = "Regex"
-let emptyProtocolName = "EmptyProtocol"
-let emptyStructName = "Empty"
+let emptyProtocolName = "EmptyCaptureProtocol"
+let baseMatchTypeName = "Substring"
 
 @main
 struct VariadicsGenerator: ParsableCommand {
@@ -114,6 +113,10 @@ struct VariadicsGenerator: ParsableCommand {
 
       """)
 
+    for arity in 2...maxArity+1 {
+      emitTupleStruct(arity: arity)
+    }
+
     for arity in minArity...maxArity {
       for permutation in Permutations(arity: arity) {
         emitConcatenation(permutation: permutation)
@@ -124,42 +127,111 @@ struct VariadicsGenerator: ParsableCommand {
     output("// END AUTO-GENERATED CONTENT")
   }
 
+  func emitTupleStruct(arity: Int) {
+    output("@frozen public struct Tuple\(arity)<")
+    outputForEach(0..<arity, separator: ", ") {
+      "_\($0)"
+    }
+    output("> {\n")
+    // `public typealias Tuple = (_0, ...)`
+    output("\n  public typealias Tuple = (")
+    outputForEach(0..<arity, separator: ", ") { "_\($0)" }
+    output(")")
+    // `public var tuple: Tuple`
+    output("\n  public var tuple: Tuple")
+    output("}\n")
+    output("extension Tuple\(arity): \(emptyProtocolName) where ")
+    outputForEach(1..<arity, separator: ", ") {
+      "_\($0): \(emptyProtocolName)"
+    }
+    output(" {}\n")
+    output("extension Tuple\(arity): MatchProtocol {\n")
+    output("  public typealias Capture = ")
+    if arity == 2 {
+      output("_1")
+    } else {
+      output("Tuple\(arity-1)<")
+      outputForEach(1..<arity, separator: ", ") {
+        "_\($0)"
+      }
+      output(">")
+    }
+    output("\n  public init(_ tuple: Tuple) { self.tuple = tuple }")
+    // `public init(_0: _0, ...) { ... }`
+    output("\n  public init(")
+    outputForEach(0..<arity, separator: ", ") {
+      "_ _\($0): _\($0)"
+    }
+    output(") {\n")
+    output("    self.init((")
+    outputForEach(0..<arity, separator: ", ") { "_\($0)" }
+    output("))")
+    output("  }")
+    output("\n}\n")
+    // Equatable
+    output("extension Tuple\(arity): Equatable where ")
+    outputForEach(0..<arity, separator: ", ") {
+      "_\($0): Equatable"
+    }
+    output(" {\n")
+    output("  public static func == (lhs: Self, rhs: Self) -> Bool {\n")
+    output("    ")
+    outputForEach(0..<arity, separator: " && ") {
+      "lhs._\($0) == rhs._\($0)"
+    }
+    output("\n  }")
+    output("}\n")
+  }
+
   func emitConcatenation(permutation: Permutation) {
     let arity = permutation.arity
+
+    func emitGenericParameters(withConstraints: Bool) {
+      outputForEach(0..<arity, separator: ", ") {
+        var base = "T\($0)"
+        if withConstraints {
+          base += ": \(patternProtocolName)"
+        }
+        return base
+      }
+    }
+
     // Emit concatenation type declarations.
     //   public struct Concatenation{n}_{perm}<...>: RegexProtocol {
-    //     public typealias Capture = ...
-    //     public let regex: Regex
+    //     public typealias Match = ...
+    //     public let regex: Regex<Match>
     //     public init(...) { ... }
     //   }
-    let typeName = "\(concatenationStructTypeBaseName)\(arity)_\(permutation.identifier)"
+    let typeName =
+      "\(concatenationStructTypeBaseName)\(arity)_\(permutation.identifier)"
     output("public struct \(typeName)<\n  ")
-    outputForEach(0..<arity, separator: ",") { "T\($0): \(patternProtocolName)" }
+    emitGenericParameters(withConstraints: true)
     output("\n>: \(patternProtocolName)")
     if permutation.hasCaptureless {
       output(" where ")
-      outputForEach(permutation.capturelessIndices, separator: ",") {
-        "T\($0).\(captureAssociatedTypeName): \(emptyProtocolName)"
+      outputForEach(permutation.capturelessIndices, separator: ", ") {
+        "T\($0).\(matchAssociatedTypeName).\(captureAssociatedTypeName): \(emptyProtocolName)"
       }
     }
     output(" {\n")
     let captureIndices = permutation.captureIndices
-    output("  public typealias \(captureAssociatedTypeName) = ")
+    output("  public typealias \(matchAssociatedTypeName) = ")
     let captureElements = captureIndices
-      .map { "T\($0).\(captureAssociatedTypeName)" }
+      .map { "T\($0).\(matchAssociatedTypeName).\(captureAssociatedTypeName)" }
     if captureElements.isEmpty {
-      output(emptyStructName)
+      output(baseMatchTypeName)
     } else {
-      output("(\(captureElements.joined(separator: ", ")))")
+      let count = captureElements.count + 1
+      output("Tuple\(count)<\(baseMatchTypeName), \(captureElements.joined(separator: ", "))>")
     }
     output("\n")
-    output("  public let \(patternProtocolRequirementName): \(PatternTypeBaseName)<\(captureAssociatedTypeName)>\n")
+    output("  public let \(patternProtocolRequirementName): \(PatternTypeBaseName)<\(matchAssociatedTypeName)>\n")
     output("  init(")
-    outputForEach(0..<arity, separator: ",") { "_ x\($0): T\($0)" }
+    outputForEach(0..<arity, separator: ", ") { "_ x\($0): T\($0)" }
     output(") {\n")
     output("    \(patternProtocolRequirementName) = .init(ast: concat(\n      ")
     outputForEach(
-      0..<arity, separator: ",", lineTerminator: ""
+      0..<arity, separator: ", ", lineTerminator: ""
     ) { i in
       "x\(i).\(patternProtocolRequirementName).ast"
     }
@@ -169,14 +241,14 @@ struct VariadicsGenerator: ParsableCommand {
     // Emit concatenation builders.
     output("extension \(patternBuilderTypeName) {\n")
     output("  public static func buildBlock<")
-    outputForEach(0..<arity, separator: ",") { "T\($0)" }
+    emitGenericParameters(withConstraints: true)
     output(">(\n    ")
-    outputForEach(0..<arity, separator: ",") { "_ x\($0): T\($0)" }
+    outputForEach(0..<arity, separator: ", ") { "_ x\($0): T\($0)" }
     output("\n  ) -> \(typeName)<")
-    outputForEach(0..<arity, separator: ",") { "T\($0)" }
+    emitGenericParameters(withConstraints: false)
     output("> {\n")
     output("    \(typeName)(")
-    outputForEach(0..<arity, separator: ",") { "x\($0)" }
+    outputForEach(0..<arity, separator: ", ") { "x\($0)" }
     output(")\n  }\n}\n\n")
   }
 }

Sources/_MatchingEngine/Regex/Parse/CaptureStructure.swift

@@ -25,7 +25,6 @@ extension AST {
         .reduce(.empty, +)
         .map(CaptureStructure.optional)
     case .concatenation(let concatenation):
-      assert(concatenation.children.count > 1)
       return concatenation.children.map(\.captureStructure).reduce(.empty, +)
     case .group(let group):
       let innerCaptures = group.child.captureStructure
@@ -107,6 +106,27 @@ extension CaptureStructure {
     }
     return false
   }
+
+  public func type(withAtomType atomType: Any.Type) -> Any.Type {
+    switch self {
+    case .atom:
+      return atomType
+    case .array(let child):
+      return TypeConstruction.arrayType(of: child.type(withAtomType: atomType))
+    case .optional(let child):
+      return TypeConstruction.optionalType(of: child.type(withAtomType: atomType))
+    case .tuple(let children):
+      return TypeConstruction.tupleType(of: children.map {
+        $0.type(withAtomType: atomType)
+      })
+    }
+  }
+
+  public typealias DefaultAtomType = Substring
+
+  public var type: Any.Type {
+    type(withAtomType: DefaultAtomType.self)
+  }
 }
 
 // MARK: - Serialization

Sources/_StringProcessing/Utility/TypeConstruction.swift renamed to Sources/_MatchingEngine/Utility/TypeConstruction.swift

@@ -48,10 +48,9 @@ private func swift_getTupleTypeMetadata3(
   proposedWitnesses: UnsafeRawPointer?
 ) -> (value: Any.Type, state: Int)
 
-enum TypeConstruction {
-
+public enum TypeConstruction {
   /// Returns a tuple metatype of the given element types.
-  static func tupleType<
+  public static func tupleType<
     ElementTypes: BidirectionalCollection
   >(
     of elementTypes: __owned ElementTypes
@@ -104,7 +103,7 @@ enum TypeConstruction {
   }
 
   /// Creates a type-erased tuple with the given elements.
-  static func tuple<Elements: BidirectionalCollection>(
+  public static func tuple<Elements: BidirectionalCollection>(
     of elements: __owned Elements
   ) -> Any where Elements.Element == Any {
     // Open existential on the overall tuple type.
@@ -133,4 +132,18 @@ enum TypeConstruction {
     let elementTypes = elements.map { type(of: $0) }
     return _openExistential(tupleType(of: elementTypes), do: create)
   }
+
+  public static func arrayType(of childType: Any.Type) -> Any.Type {
+    func helper<T>(_: T.Type) -> Any.Type {
+      [T].self
+    }
+    return _openExistential(childType, do: helper)
+  }
+
+  public static func optionalType(of childType: Any.Type) -> Any.Type {
+    func helper<T>(_: T.Type) -> Any.Type {
+      T?.self
+    }
+    return _openExistential(childType, do: helper)
+  }
 }

Sources/_StringProcessing/Capture.swift

@@ -5,8 +5,17 @@ import _MatchingEngine
 enum Capture {
   case atom(Any)
   indirect case tuple([Capture])
-  indirect case optional(Capture?)
-  indirect case array([Capture])
+  indirect case some(Capture)
+  case none(childType: AnyCaptureType)
+  indirect case array([Capture], childType: AnyCaptureType)
+
+  static func none(childType: Any.Type) -> Capture {
+    .none(childType: AnyCaptureType(childType))
+  }
+
+  static func array(_ children: [Capture], childType: Any.Type) -> Capture {
+    .array(children, childType: AnyCaptureType(childType))
+  }
 }
 
 extension Capture {
@@ -25,21 +34,48 @@ extension Capture {
     case .tuple(let elements):
       return TypeConstruction.tuple(
         of: elements.map(\.value))
-    case .array(let elements):
-      guard let first = elements.first else {
-        return [Any]()
+    case .array(let elements, let childType):
+      func helper<T>(_: T.Type) -> Any {
+        elements.map { $0.value as! T }
       }
-      // When the array is not empty, infer the concrete `Element `type from the first element.
-      func helper<T>(_ first: T) -> Any {
-        var castElements = [first]
-        for element in elements.dropFirst() {
-          castElements.append(element.value as! T)
-        }
-        return castElements
+      return _openExistential(childType.base, do: helper)
+    case .some(let subcapture):
+      return subcapture.value
+    case .none(let childType):
+      func helper<T>(_: T.Type) -> Any {
+        nil as T? as Any
       }
-      return _openExistential(first.value, do: helper)
-    case .optional(let subcapture):
-      return subcapture?.value as Any
+      return _openExistential(childType.base, do: helper)
+    }
+  }
+
+  private func prepending(_ newElement: Any) -> Self {
+    switch self {
+    case .atom, .some, .none, .array:
+      return .tuple([.atom(newElement), self])
+    case .tuple(let elements):
+      return .tuple([.atom(newElement)] + elements)
     }
   }
+
+  func matchValue(withWholeMatch wholeMatch: Substring) -> Any {
+    prepending(wholeMatch).value
+  }
+}
+
+/// A wrapper of an existential metatype, equatable and hashable by reference.
+struct AnyCaptureType: Equatable, Hashable {
+  var base: Any.Type
+
+  init(_ type: Any.Type) {
+    base = type
+  }
+
+  static func == (lhs: AnyCaptureType, rhs: AnyCaptureType) -> Bool {
+    lhs.base == rhs.base
+  }
+
+  func hash(into hasher: inout Hasher) {
+    hasher.combine(ObjectIdentifier(base))
+  }
 }