WIP: structuralize captures

Author: milseman

Sources/_StringProcessing/ByteCodeGen.swift

@@ -248,13 +248,13 @@ extension Compiler.ByteCodeGen {
 
   mutating func emitGroup(
     _ kind: AST.Group.Kind, _ child: DSLTree.Node
-  ) throws {
+  ) throws -> CaptureRegister? {
     options.beginScope()
     defer { options.endScope() }
 
     if let lookaround = kind.lookaroundKind {
       try emitLookaround(lookaround, child)
-      return
+      return nil
     }
 
     switch kind {
@@ -267,14 +267,17 @@ extension Compiler.ByteCodeGen {
       builder.buildBeginCapture(cap)
       try emitNode(child)
       builder.buildEndCapture(cap)
+      return cap
 
     case .changeMatchingOptions(let optionSequence, _):
       options.apply(optionSequence)
       try emitNode(child)
+      return nil
 
     default:
       // FIXME: Other kinds...
       try emitNode(child)
+      return nil
     }
   }
 
@@ -494,7 +497,7 @@ extension Compiler.ByteCodeGen {
       }
 
     case let .group(kind, child):
-      try emitGroup(kind, child)
+      _ = try emitGroup(kind, child)
 
     case .conditional:
       throw Unsupported("Conditionals")
@@ -518,9 +521,21 @@ extension Compiler.ByteCodeGen {
     case let .convertedRegexLiteral(n, _):
       try emitNode(n)
 
-    case let .groupTransform(kind, child, _):
-      try emitGroup(kind, child)
-      // FIXME: Transforms
+    case let .groupTransform(kind, child, t):
+      guard let cap = try emitGroup(kind, child) else {
+        assertionFailure("""
+          What does it mean to not have a capture to transform?
+          """)
+        return
+      }
+
+      // FIXME: Is this how we want to do it?
+      let transform = builder.makeTransformFunction {
+        input, range in
+        t(input[range])
+      }
+
+      builder.buildTransformCapture(cap, transform)
 
     case .absentFunction:
       throw Unsupported("absent function")

Sources/_StringProcessing/Capture.swift

@@ -19,7 +19,9 @@ enum Capture {
   indirect case some(Capture)
   case none(childType: AnyType)
   indirect case array([Capture], childType: AnyType)
+}
 
+extension Capture {
   static func none(childType: Any.Type) -> Capture {
     .none(childType: AnyType(childType))
   }
@@ -101,7 +103,7 @@ extension Capture: CustomStringConvertible {
       }
 
     case let .some(n):
-      printer.printBlock("Tuple") { printer in
+      printer.printBlock("Some") { printer in
         n._print(&printer)
       }
 

Sources/_StringProcessing/Compiler.swift

@@ -28,6 +28,7 @@ class Compiler {
   __consuming func emit() throws -> Program {
     // TODO: Handle global options
     var codegen = ByteCodeGen(options: options)
+    codegen.builder.captureStructure = tree.captureStructure
     try codegen.emitNode(tree.root)
     let program = try codegen.finish()
     return program

Sources/_StringProcessing/Engine/Consume.swift

@@ -62,7 +62,7 @@ extension Engine where Input == String {
     }
     guard let result = result else { return nil }
 
-    let capList = cpu.storedCaptures.map { $0.history }
+    let capList = cpu.storedCaptures
     return (result, CaptureList(caps: capList))
   }
 }

Sources/_StringProcessing/Engine/InstPayload.swift

@@ -61,6 +61,7 @@ extension Instruction.Payload {
     case packedBoolInt(BoolRegister, IntRegister)
     case packedEltBool(ElementRegister, BoolRegister)
     case packedPosPos(PositionRegister, PositionRegister)
+    case packedCapTran(CaptureRegister, TransformRegister)
   }
 }
 
@@ -280,5 +281,13 @@ extension Instruction.Payload {
     interpretPair()
   }
 
+  init(capture: CaptureRegister, transform: TransformRegister) {
+    self.init(capture, transform)
+  }
+  var pairedCaptureTransform: (
+    CaptureRegister, TransformRegister
+  ) {
+    interpretPair()
+  }
 }
 

Sources/_StringProcessing/Engine/Instruction.swift

@@ -240,6 +240,12 @@ extension Instruction {
     ///
     case endCapture
 
+    /// Transform a captured value, saving the built value
+    ///
+    ///     transformCapture(_:CapReg, _:TransformReg)
+    ///
+    case transformCapture
+
     /// Match a previously captured value
     ///
     ///     backreference(_:CapReg)

Sources/_StringProcessing/Engine/MEBuilder.swift

@@ -18,8 +18,10 @@ extension MEProgram where Input.Element: Hashable {
     var elements = TypedSetVector<Input.Element, _ElementRegister>()
     var sequences = TypedSetVector<[Input.Element], _SequenceRegister>()
     var strings = TypedSetVector<String, _StringRegister>()
+
     var consumeFunctions: [ConsumeFunction] = []
     var assertionFunctions: [AssertionFunction] = []
+    var transformFunctions: [TransformFunction] = []
 
     // Map tokens to actual addresses
     var addressTokens: [InstructionAddress?] = []
@@ -34,6 +36,10 @@ extension MEProgram where Input.Element: Hashable {
     // Special addresses or instructions
     var failAddressToken: AddressToken? = nil
 
+    // TODO: Should we have better API for building this up
+    // as we compile?
+    var captureStructure: CaptureStructure = .empty
+
     public init() {}
   }
 }
@@ -223,6 +229,14 @@ extension MEProgram.Builder {
       .init(.endCapture, .init(capture: cap)))
   }
 
+  public mutating func buildTransformCapture(
+    _ cap: CaptureRegister, _ trans: TransformRegister
+  ) {
+    instructions.append(.init(
+      .transformCapture,
+      .init(capture: cap, transform: trans)))
+  }
+
   public mutating func buildBackreference(
     _ cap: CaptureRegister
   ) {
@@ -286,6 +300,7 @@ extension MEProgram.Builder {
     regInfo.positions = nextPositionRegister.rawValue
     regInfo.consumeFunctions = consumeFunctions.count
     regInfo.assertionFunctions = assertionFunctions.count
+    regInfo.transformFunctions = transformFunctions.count
     regInfo.captures = nextCaptureRegister.rawValue
 
     return MEProgram(
@@ -295,7 +310,9 @@ extension MEProgram.Builder {
       staticStrings: strings.stored,
       staticConsumeFunctions: consumeFunctions,
       staticAssertionFunctions: assertionFunctions,
-      registerInfo: regInfo)
+      staticTransformFunctions: transformFunctions,
+      registerInfo: regInfo,
+      captureStructure: captureStructure)
   }
 
   public mutating func reset() { self = Self() }
@@ -433,5 +450,11 @@ extension MEProgram.Builder {
     defer { assertionFunctions.append(f) }
     return AssertionFunctionRegister(assertionFunctions.count)
   }
+  public mutating func makeTransformFunction(
+    _ f: @escaping MEProgram.TransformFunction
+  ) -> TransformRegister {
+    defer { transformFunctions.append(f) }
+    return TransformRegister(transformFunctions.count)
+  }
 }
 

Sources/_StringProcessing/Engine/MECapture.swift

@@ -9,6 +9,8 @@
 //
 //===----------------------------------------------------------------------===//
 
+import _MatchingEngine
+
 /*
 
  TODO: Specialized data structure for all captures:
@@ -45,28 +47,45 @@ extension Processor {
     //
     fileprivate var stack: Array<Range<Position>> = []
 
+    // Also save entire history of captured values -_-
+    //
+    // We will need to really zoom in on performance here...
+    fileprivate var valueStack: Array<Any> = []
+
     // An in-progress capture start
     fileprivate var currentCaptureBegin: Position? = nil
 
     fileprivate func _invariantCheck() {
       if startState == nil {
         assert(stack.isEmpty)
+        assert(valueStack.isEmpty)
         assert(currentCaptureBegin == nil)
       } else {
         assert(!stack.isEmpty || currentCaptureBegin != nil)
       }
+      if hasValues {
+        // FIXME: how?
+        // assert(valueStack.count == stack.count)
+      }
     }
 
     // MARK: - IPI
 
     var isEmpty: Bool { stack.isEmpty }
 
+    var hasValues: Bool { !valueStack.isEmpty }
+
     var history: Array<Range<Position>> {
       stack
     }
+    var valueHistory: Array<Any> {
+      valueStack
+    }
 
     var latest: Range<Position>? { stack.last }
 
+    var latestValue: Any? { valueStack.last }
+
     /// Start a new capture. If the previously started one was un-ended,
     /// will clear it and restart. If this is the first start, will save `initial`.
     mutating func startCapture(
@@ -89,6 +108,14 @@ extension Processor {
       stack.append(currentCaptureBegin! ..< idx)
     }
 
+    mutating func registerValue(
+      _ value: Any
+    ) {
+      _invariantCheck()
+      defer { _invariantCheck() }
+      valueStack.append(value)
+    }
+
     mutating func fail(truncatingAt stackIdx: Int) {
       _invariantCheck()
       assert(stackIdx <= stack.endIndex)
@@ -102,15 +129,28 @@ extension Processor {
   }
 }
 
-public struct CaptureList {
-  var caps: Array<Array<Range<String.Index>>>
-
-  func extract(from s: String) -> Array<Array<Substring>> {
-    caps.map { $0.map { s[$0] }  }
+extension Processor._StoredCapture: CustomStringConvertible {
+  var description: String {
+    if hasValues {
+      return String(describing: valueStack)
+    }
+    return String(describing: history)
   }
+}
 
-  func latest(from s: String) -> Array<Substring?> {
-    // TODO: If empty, probably need empty range or something...
-    extract(from: s).map { $0.last }
+public struct CaptureList {
+  var caps: Array<Processor<String>._StoredCapture>
+
+//  func extract(from s: String) -> Array<Array<Substring>> {
+//    caps.map { $0.map { s[$0] }  }
+//  }
+//
+  func latestUntyped(from s: String) -> Array<Substring?> {
+    caps.map {
+      guard let last = $0.latest else {
+        return nil
+      }
+      return s[last]
+    }
   }
 }

Sources/_StringProcessing/Engine/MEProgram.swift

@@ -9,21 +9,29 @@
 //
 //===----------------------------------------------------------------------===//
 
+import _MatchingEngine
+
 public struct MEProgram<Input: Collection> where Input.Element: Equatable {
   public typealias ConsumeFunction = (Input, Range<Input.Index>) -> Input.Index?
   public typealias AssertionFunction =
     (Input, Input.Index, Range<Input.Index>) -> Bool
+  public typealias TransformFunction =
+    (Input, Range<Input.Index>) -> Any?
+
   var instructions: InstructionList<Instruction>
 
   var staticElements: [Input.Element]
   var staticSequences: [[Input.Element]]
   var staticStrings: [String]
   var staticConsumeFunctions: [ConsumeFunction]
   var staticAssertionFunctions: [AssertionFunction]
+  var staticTransformFunctions: [TransformFunction]
 
   var registerInfo: RegisterInfo
 
   var enableTracing: Bool = false
+
+  let captureStructure: CaptureStructure
 }
 
 extension MEProgram: CustomStringConvertible {

Sources/_StringProcessing/Engine/Processor.swift

@@ -406,6 +406,23 @@ extension Processor {
        storedCaptures[capNum].endCapture(currentPosition)
        controller.step()
 
+    case .transformCapture:
+      let (cap, trans) = payload.pairedCaptureTransform
+      let transform = registers[trans]
+      let capNum = Int(asserting: cap.rawValue)
+
+      guard let range = storedCaptures[capNum].latest else {
+        fatalError(
+          "Unreachable: transforming without a capture")
+      }
+      // FIXME: Pass input or the slice?
+      guard let value = transform(input, range) else {
+        signalFailure()
+        return
+      }
+      storedCaptures[capNum].registerValue(value)
+
+      controller.step()
     }
   }
 }