Track the whole match as a capture in the matching engine.

Author: rxwei

Sources/_RegexParser/Regex/Parse/CaptureList.swift

@@ -125,6 +125,7 @@ extension AST.Node {
 
   public var _captureList: CaptureList {
     var caps = CaptureList()
+    caps.append(.init(type: Substring.self, optionalDepth: 0, .fake))
     self._addCaptures(to: &caps, optionalNesting: 0)
     return caps
   }

Sources/_RegexParser/Regex/Parse/Sema.swift

@@ -77,7 +77,7 @@ extension RegexValidator {
     }
     switch ref.kind {
     case .absolute(let i):
-      guard i <= captures.captures.count else {
+      guard i < captures.captures.count else {
         throw error(.invalidReference(i), at: ref.innerLoc)
       }
     case .named(let name):

Sources/_StringProcessing/ByteCodeGen.swift

@@ -4,21 +4,33 @@ extension Compiler {
   struct ByteCodeGen {
     var options: MatchingOptions
     var builder = Program.Builder()
+    let wholeMatchRegister: CaptureRegister
 
     init(options: MatchingOptions, captureList: CaptureList) {
       self.options = options
       self.builder.captureList = captureList
+      // The first output element is the whole match.
+      wholeMatchRegister = self.builder.makeCapture(id: nil, name: nil)
     }
+  }
+}
 
-    mutating func finish(
-    ) throws -> Program {
-      builder.buildAccept()
-      return try builder.assemble()
+extension Compiler.ByteCodeGen {
+  mutating func emitRoot(_ root: DSLTree.Node) throws -> Program {
+    switch root {
+    case .matcher(_, let f):
+      emitMatcher(f, into: wholeMatchRegister)
+    default:
+      builder.buildBeginCapture(wholeMatchRegister)
+      try emitNode(root)
+      builder.buildEndCapture(wholeMatchRegister)
     }
+    builder.buildAccept()
+    return try builder.assemble()
   }
 }
 
-extension Compiler.ByteCodeGen {
+fileprivate extension Compiler.ByteCodeGen {
   mutating func emitAtom(_ a: DSLTree.Atom) throws {
     switch a {
     case .any:
@@ -65,8 +77,7 @@ extension Compiler.ByteCodeGen {
 
     switch ref.kind {
     case .absolute(let i):
-      // Backreferences number starting at 1
-      builder.buildBackreference(.init(i-1))
+      builder.buildBackreference(.init(i))
     case .named(let name):
       try builder.buildNamedReference(name)
     case .relative:
@@ -729,4 +740,3 @@ extension Compiler.ByteCodeGen {
     }
   }
 }
-

Sources/_StringProcessing/Capture.swift

@@ -13,7 +13,7 @@
 
 // TODO: Where should this live? Inside TypeConstruction?
 func constructExistentialOutputComponent(
-  from input: Substring,
+  from input: String,
   in range: Range<String.Index>?,
   value: Any?,
   optionalCount: Int
@@ -43,7 +43,7 @@ func constructExistentialOutputComponent(
 @available(SwiftStdlib 5.7, *)
 extension AnyRegexOutput.Element {
   func existentialOutputComponent(
-    from input: Substring
+    from input: String
   ) -> Any {
     constructExistentialOutputComponent(
       from: input,
@@ -64,15 +64,13 @@ extension Sequence where Element == AnyRegexOutput.Element {
   // FIXME: This is a stop gap where we still slice the input
   // and traffic through existentials
   @available(SwiftStdlib 5.7, *)
-  func existentialOutput(
-    from input: Substring
-  ) -> Any {
-    var caps = Array<Any>()
-    caps.append(input)
-    caps.append(contentsOf: self.map {
+  func existentialOutput(from input: String) -> Any {
+    let elements = map {
       $0.existentialOutputComponent(from: input)
-    })
-    return TypeConstruction.tuple(of: caps)
+    }
+    return elements.count == 1
+      ? elements[0]
+      : TypeConstruction.tuple(of: elements)
   }
 
   func slices(from input: String) -> [Substring?] {

Sources/_StringProcessing/Compiler.swift

@@ -30,9 +30,7 @@ class Compiler {
     var codegen = ByteCodeGen(
       options: options, captureList: tree.root._captureList
     )
-    try codegen.emitNode(tree.root)
-    let program = try codegen.finish()
-    return program
+    return try codegen.emitRoot(tree.root)
   }
 }
 

Sources/_StringProcessing/Executor.swift

@@ -62,8 +62,7 @@ struct Executor {
 
     return .init(
       anyRegexOutput: anyRegexOutput,
-      range: range,
-      value: value
+      range: range
     )
   }
 

Sources/_StringProcessing/Regex/AnyRegexOutput.swift

@@ -15,7 +15,7 @@
 @available(SwiftStdlib 5.7, *)
 public struct AnyRegexOutput {
   internal let input: String
-  internal let _elements: [ElementRepresentation]
+  internal var _elements: [ElementRepresentation]
 }
 
 @available(SwiftStdlib 5.7, *)
@@ -37,7 +37,7 @@ extension AnyRegexOutput {
     as type: Output.Type = Output.self
   ) -> Output? {
     let elements = map {
-      $0.existentialOutputComponent(from: input[...])
+      $0.existentialOutputComponent(from: input)
     }
     return TypeConstruction.tuple(of: elements) as? Output
   }
@@ -194,8 +194,7 @@ extension Regex.Match where Output == AnyRegexOutput {
   public init<Output>(_ match: Regex<Output>.Match) {
     self.init(
       anyRegexOutput: match.anyRegexOutput,
-      range: match.range,
-      value: match.value
+      range: match.range
     )
   }
 }
@@ -250,12 +249,12 @@ extension AnyRegexOutput {
 @available(SwiftStdlib 5.7, *)
 extension AnyRegexOutput.ElementRepresentation {
   fileprivate func value(forInput input: String) -> Any {
-    // Ok for now because `existentialMatchComponent`
-    // wont slice the input if there's no range to slice with
-    //
-    // FIXME: This is ugly :-/
-    let input = bounds.map { input[$0] } ?? ""
-
+//    // Ok for now because `existentialMatchComponent`
+//    // wont slice the input if there's no range to slice with
+//    //
+//    // FIXME: This is ugly :-/
+//    let input = bounds.map { input[$0] } ?? ""
+//
     return constructExistentialOutputComponent(
       from: input,
       in: bounds,

Sources/_StringProcessing/Regex/DSLTree.swift

@@ -524,9 +524,17 @@ extension DSLTree.Node {
 
   var _captureList: CaptureList {
     var list = CaptureList()
+    list.append(.init(type: wholeMatchType, optionalDepth: 0, .fake))
     self._addCaptures(to: &list, optionalNesting: 0)
     return list
   }
+
+  var wholeMatchType: Any.Type {
+    if case .matcher(let type, _) = self {
+      return type
+    }
+    return Substring.self
+  }
 }
 
 extension DSLTree {

Sources/_StringProcessing/Regex/Match.swift

@@ -21,8 +21,6 @@ extension Regex {
 
     /// The range of the overall match.
     public let range: Range<String.Index>
-
-    let value: Any?
   }
 }
 
@@ -31,34 +29,12 @@ extension Regex.Match {
   /// The output produced from the match operation.
   public var output: Output {
     if Output.self == AnyRegexOutput.self {
-      let wholeMatchCapture = AnyRegexOutput.ElementRepresentation(
-        optionalDepth: 0,
-        bounds: range
-      )
-      
-      let output = AnyRegexOutput(
-        input: anyRegexOutput.input,
-        elements: [wholeMatchCapture] + anyRegexOutput._elements
-      )
-      
-      return output as! Output
-    } else if Output.self == Substring.self {
-      // FIXME: Plumb whole match (`.0`) through the matching engine.
-      return anyRegexOutput.input[range] as! Output
-    } else if anyRegexOutput.isEmpty, value != nil {
-      // FIXME: This is a workaround for whole-match values not
-      // being modeled as part of captures. We might want to
-      // switch to a model where results are alongside captures
-      return value! as! Output
-    } else {
-      guard value == nil else {
-        fatalError("FIXME: what would this mean?")
-      }
-      let typeErasedMatch = anyRegexOutput.existentialOutput(
-        from: anyRegexOutput.input[range]
-      )
-      return typeErasedMatch as! Output
+      return anyRegexOutput as! Output
     }
+    let typeErasedMatch = anyRegexOutput.existentialOutput(
+      from: anyRegexOutput.input
+    )
+    return typeErasedMatch as! Output
   }
 
   /// Accesses a capture by its name or number.
@@ -83,7 +59,7 @@ extension Regex.Match {
     }
 
     return element.existentialOutputComponent(
-      from: anyRegexOutput.input[...]
+      from: anyRegexOutput.input
     ) as! Capture
   }
 }

Sources/_StringProcessing/Utility/TypeVerification.swift

@@ -18,8 +18,8 @@ extension Regex {
       return true
     }
 
-    var tupleElements: [Any.Type] = [Substring.self]
-    var labels = " "
+    var tupleElements: [Any.Type] = []
+    var labels = ""
 
     for capture in program.tree.root._captureList.captures {
       var captureType: Any.Type = capture.type ?? Substring.self
@@ -41,7 +41,7 @@ extension Regex {
 
     // If we have no captures, then our Regex must be Regex<Substring>.
     if tupleElements.count == 1 {
-      return Output.self == Substring.self
+      return Output.self == program.tree.root.wholeMatchType
     }
 
     let createdType = TypeConstruction.tupleType(

Tests/RegexTests/CaptureTests.swift

@@ -159,7 +159,9 @@ func captureTest(
   line: UInt = #line
 ) {
   let ast = try! parse(regex, .semantic, .traditional)
-  let capList = ast.root._captureList.withoutLocs
+  var capList = ast.root._captureList.withoutLocs
+  // Peel off the whole match element.
+  capList.captures.removeFirst()
   guard capList == expected else {
     XCTFail("""
       Expected:
@@ -173,7 +175,9 @@ func captureTest(
   }
 
   // Ensure DSLTree preserves literal captures
-  let dslCapList = ast.dslTree.root._captureList
+  var dslCapList = ast.dslTree.root._captureList
+  // Peel off the whole match element.
+  dslCapList.captures.removeFirst()
   guard dslCapList == capList else {
     XCTFail("""
       DSLTree did not preserve structure:
@@ -202,7 +206,9 @@ func captureTest(
       return
     }
 
-    let caps = result.anyRegexOutput
+    var caps = result.anyRegexOutput
+    // Peel off the whole match element.
+    caps._elements.removeFirst()
     guard caps.count == output.count else {
       XCTFail("""
       Mismatch capture count:

Tests/RegexTests/MatchTests.swift

@@ -74,7 +74,7 @@ func flatCaptureTest(
 ) {
   for (test, expect) in tests {
     do {
-      guard let (_, caps) = try? _firstMatch(
+      guard var (_, caps) = try? _firstMatch(
         regex,
         input: test,
         syntax: syntax,
@@ -86,6 +86,8 @@ func flatCaptureTest(
           throw MatchError("Match failed")
         }
       }
+      // Peel off the whole match.
+      caps.removeFirst()
       guard let expect = expect else {
         throw MatchError("""
             Match of \(test) succeeded where failure expected in \(regex)

Tests/RegexTests/ParseTests.swift

@@ -94,7 +94,9 @@ func parseTest(
             file: file, line: line)
     return
   }
-  let captures = ast.captureList.withoutLocs
+  var captures = ast.captureList.withoutLocs
+  // Peel off the whole match.
+  captures.captures.removeFirst()
   guard captures == expectedCaptures else {
     XCTFail("""