Generalized bidirectional assertion support

Adds generalized assertion support as well support for most
built-in assertions and anchors.

Author: milseman

Sources/_MatchingEngine/Engine/Builder.swift

@@ -6,6 +6,7 @@ extension Program where Input.Element: Hashable {
     var sequences = TypedSetVector<[Input.Element], _SequenceRegister>()
     var strings = TypedSetVector<String, _StringRegister>()
     var consumeFunctions: [ConsumeFunction] = []
+    var assertionFunctions: [AssertionFunction] = []
 
     // Map tokens to actual addresses
     var addressTokens: [InstructionAddress?] = []
@@ -173,6 +174,13 @@ extension Program.Builder {
       .consumeBy, .init(consumer: makeConsumeFunction(p))))
   }
 
+  public mutating func buildAssert(
+    by p: @escaping Program.AssertionFunction
+  ) {
+    instructions.append(.init(
+      .assertBy, .init(assertion: makeAssertionFunction(p))))
+  }
+
   public mutating func buildAssert(
     _ e: Input.Element, into cond: BoolRegister
   ) {
@@ -243,13 +251,15 @@ extension Program.Builder {
     regInfo.ints = nextIntRegister.rawValue
     regInfo.positions = nextPositionRegister.rawValue
     regInfo.consumeFunctions = consumeFunctions.count
+    regInfo.assertionFunctions = assertionFunctions.count
 
     return Program(
       instructions: InstructionList(instructions),
       staticElements: elements.stored,
       staticSequences: sequences.stored,
       staticStrings: strings.stored,
       staticConsumeFunctions: consumeFunctions,
+      staticAssertionFunctions: assertionFunctions,
       registerInfo: regInfo)
   }
 
@@ -377,5 +387,11 @@ extension Program.Builder {
     defer { consumeFunctions.append(f) }
     return ConsumeFunctionRegister(consumeFunctions.count)
   }
+  public mutating func makeAssertionFunction(
+    _ f: @escaping Program.AssertionFunction
+  ) -> AssertionFunctionRegister {
+    defer { assertionFunctions.append(f) }
+    return AssertionFunctionRegister(assertionFunctions.count)
+  }
 }
 

Sources/_MatchingEngine/Engine/Engine.swift

@@ -1,6 +1,6 @@
 // Currently, engine binds the type and consume binds an instance.
 // But, we can play around with this.
-public struct Engine<Input: Collection> where Input.Element: Hashable {
+public struct Engine<Input: BidirectionalCollection> where Input.Element: Hashable {
 
   var program: Program<Input>
 

Sources/_MatchingEngine/Engine/InstPayload.swift

@@ -39,6 +39,7 @@ extension Instruction.Payload {
     case bool(BoolRegister)
     case element(ElementRegister)
     case consumer(ConsumeFunctionRegister)
+    case assertion(AssertionFunctionRegister)
     case addr(InstructionAddress)
 
     case packedImmInt(Int, IntRegister)
@@ -186,6 +187,13 @@ extension Instruction.Payload {
     interpret()
   }
 
+  init(assertion: AssertionFunctionRegister) {
+    self.init(assertion)
+  }
+  var assertion: AssertionFunctionRegister {
+    interpret()
+  }
+
   init(addr: InstructionAddress) {
     self.init(addr)
   }

Sources/_MatchingEngine/Engine/Instruction.swift

@@ -162,11 +162,18 @@ extension Instruction {
     /// Operand: Consume function register to call.
     case consumeBy
 
-    /// Custom assertion operation
+    /// Custom lookaround assertion operation.
+    /// Triggers a failure if customFunction returns false.
+    ///
+    ///     assert(
+    ///       _ customFunction: (
+    ///         input: Input,
+    ///         currentPos: Position,
+    ///         bounds: Range<Position>) -> Bool
+    ///     )
     ///
     /// Operands: destination bool register, assert hook register
-    static var assertHook: OpCode { fatalError() }
-
+    case assertBy
 
     // MARK: Matching: Save points
 

Sources/_MatchingEngine/Engine/Processor.swift

@@ -14,7 +14,7 @@ struct Controller {
 }
 
 struct Processor<
-  Input: Collection
+  Input: BidirectionalCollection
 > where Input.Element: Equatable { // maybe Hashable?
   typealias Element = Input.Element
 
@@ -347,6 +347,15 @@ extension Processor {
       advance(to: nextIndex)
       controller.step()
 
+    case .assertBy:
+      let reg = payload.assertion
+      let assertion = registers[reg]
+      guard assertion(input, currentPosition, bounds) else {
+        signalFailure()
+        return
+      }
+      controller.step()
+
     case .print:
       // TODO: Debug stream
       doPrint(registers[payload.string])

Sources/_MatchingEngine/Engine/Program.swift

@@ -1,11 +1,14 @@
 public struct Program<Input: Collection> where Input.Element: Equatable {
   public typealias ConsumeFunction = (Input, Range<Input.Index>) -> Input.Index?
+  public typealias AssertionFunction =
+    (Input, Input.Index, Range<Input.Index>) -> Bool
   var instructions: InstructionList<Instruction>
 
   var staticElements: [Input.Element]
   var staticSequences: [[Input.Element]]
   var staticStrings: [String]
   var staticConsumeFunctions: [ConsumeFunction]
+  var staticAssertionFunctions: [AssertionFunction]
 
   var registerInfo: RegisterInfo
 

Sources/_MatchingEngine/Engine/Registers.swift

@@ -15,6 +15,9 @@ extension Processor {
     // currently, these are static readonly
     var consumeFunctions: [Program<Input>.ConsumeFunction]
 
+    // currently, these are static readonly
+    var assertionFunctions: [Program<Input>.AssertionFunction]
+
     // currently, these are for comments and abort messages
     var strings: [String]
 
@@ -63,6 +66,9 @@ extension Processor {
     subscript(_ i: ConsumeFunctionRegister) -> Program<Input>.ConsumeFunction {
       consumeFunctions[i.rawValue]
     }
+    subscript(_ i: AssertionFunctionRegister) -> Program<Input>.AssertionFunction {
+      assertionFunctions[i.rawValue]
+    }
   }
 }
 
@@ -82,6 +88,9 @@ extension Processor.Registers {
     self.consumeFunctions = program.staticConsumeFunctions
     assert(consumeFunctions.count == info.consumeFunctions)
 
+    self.assertionFunctions = program.staticAssertionFunctions
+    assert(assertionFunctions.count == info.assertionFunctions)
+
     self.strings = program.staticStrings
     assert(strings.count == info.strings)
 
@@ -110,6 +119,7 @@ extension Program {
     var bools = 0
     var strings = 0
     var consumeFunctions = 0
+    var assertionFunctions = 0
     var ints = 0
     var floats = 0
     var positions = 0

Sources/_MatchingEngine/Regex/AST/Atom.swift

@@ -441,3 +441,65 @@ extension AST.Atom {
     }
   }
 }
+
+extension AST.Atom {
+  /// Anchors and other built-in zero-width assertions
+  public enum AssertionKind: String {
+    /// \A
+    case startOfSubject = #"\A"#
+
+    /// \Z
+    case endOfSubjectBeforeNewline = #"\Z"#
+
+    /// \z
+    case endOfSubject = #"\z"#
+
+    /// \K
+    case resetStartOfMatch = #"\K"#
+
+    /// \G
+    case firstMatchingPositionInSubject = #"\G"#
+
+    /// \y
+    case textSegment = #"\y"#
+
+    /// \Y
+    case notTextSegment = #"\Y"#
+
+    /// ^
+    case startOfLine = #"^"#
+
+    /// $
+    case endOfLine = #"$"#
+
+    /// \b (from outside a custom character class)
+    case wordBoundary = #"\b"#
+
+    /// \B
+    case notWordBoundary = #"\B"#
+
+  }
+
+  public var assertionKind: AssertionKind? {
+    switch kind {
+    case .startOfLine:     return .startOfLine
+    case .endOfLine:       return .endOfLine
+
+    case .escaped(.wordBoundary):    return .wordBoundary
+    case .escaped(.notWordBoundary): return .notWordBoundary
+    case .escaped(.startOfSubject):  return .startOfSubject
+    case .escaped(.endOfSubject):    return .endOfSubject
+    case .escaped(.textSegment):     return .textSegment
+    case .escaped(.notTextSegment):  return .notTextSegment
+    case .escaped(.endOfSubjectBeforeNewline):
+        return .endOfSubjectBeforeNewline
+    case .escaped(.firstMatchingPositionInSubject):
+        return .firstMatchingPositionInSubject
+
+    case .escaped(.resetStartOfMatch): return .resetStartOfMatch
+
+    default: return nil
+    }
+  }
+}
+

Sources/_MatchingEngine/Utility/TypedInt.swift

@@ -144,6 +144,10 @@ public enum _StringRegister {}
 public typealias ConsumeFunctionRegister = TypedInt<_ConsumeFunctionRegister>
 public enum _ConsumeFunctionRegister {}
 
+/// Used for assertion functions, e.g. anchors etc
+public typealias AssertionFunctionRegister = TypedInt<_AssertionFunctionRegister>
+public enum _AssertionFunctionRegister {}
+
 /// UNIMPLEMENTED
 public typealias IntRegister = TypedInt<_IntRegister>
 public enum _IntRegister {}

Sources/_StringProcessing/CharacterClass.swift

@@ -421,3 +421,31 @@ extension AST.CustomCharacterClass {
     return self.isInverted ? cc.inverted : cc
   }
 }
+
+extension CharacterClass {
+  // FIXME: Calling on inverted sets wont be the same as the
+  // inverse of a boundary if at the start or end of the
+  // string. (Think through what we want: do it ourselves or
+  // give the caller both options).
+  func isBoundary(
+    _ input: String,
+    at pos: String.Index,
+    bounds: Range<String.Index>
+  ) -> Bool {
+    // FIXME: How should we handle bounds?
+    // We probably need two concepts
+    if input.isEmpty { return false }
+    if pos == input.startIndex {
+      return self.matches(in: input, at: pos) != nil
+    }
+    let priorIdx = input.index(before: pos)
+    if pos == input.endIndex {
+      return self.matches(in: input, at: priorIdx) != nil
+    }
+
+    let prior = self.matches(in: input, at: priorIdx) != nil
+    let current = self.matches(in: input, at: pos) != nil
+    return prior != current
+  }
+
+}

Sources/_StringProcessing/Compiler.swift

@@ -108,11 +108,86 @@ class Compiler {
       // We stick quoted content into read-only constant strings
       builder.buildMatchSequence(q.literal)
 
+    case .atom(let a) where a.assertionKind != nil:
+      try emitAssertion(a.assertionKind!)
+
     case .customCharacterClass, .atom:
       throw unsupported(node._dumpBase)
     }
   }
 
+  func emitAssertion(_ kind: AST.Atom.AssertionKind) throws {
+    // FIXME: Depends on API model we have... We may want to
+    // think through some of these with API interactions in mind
+    //
+    // This might break how we use `bounds` for both slicing
+    // and things like `firstIndex`, that is `firstIndex` may
+    // need to supply both a slice bounds and a per-search bounds.
+    switch kind {
+    case .startOfSubject:
+      builder.buildAssert { (input, pos, bounds) in
+        pos == input.startIndex
+      }
+
+    case .endOfSubjectBeforeNewline:
+      builder.buildAssert { (input, pos, bounds) in
+        if pos == input.endIndex { return true }
+        return input.index(after: pos) == input.endIndex
+         && input[pos].isNewline
+      }
+
+    case .endOfSubject:
+      builder.buildAssert { (input, pos, bounds) in
+        pos == input.endIndex
+      }
+
+    case .resetStartOfMatch:
+      // FIXME: Figure out how to communicate this out
+      throw unsupported(#"\K (reset/keep assertion)"#)
+
+    case .firstMatchingPositionInSubject:
+      // TODO: We can probably build a nice model with API here
+      builder.buildAssert { (input, pos, bounds) in
+        pos == bounds.lowerBound
+      }
+
+    case .textSegment:
+      // This we should be able to do!
+      throw unsupported(#"\y (text segment)"#)
+
+    case .notTextSegment:
+      // This we should be able to do!
+      throw unsupported(#"\Y (not text segment)"#)
+
+    case .startOfLine:
+      builder.buildAssert { (input, pos, bounds) in
+        pos == input.startIndex ||
+        input[input.index(before: pos)].isNewline
+      }
+
+    case .endOfLine:
+      builder.buildAssert { (input, pos, bounds) in
+        pos == input.endIndex || input[pos].isNewline
+      }
+
+    case .wordBoundary:
+      // TODO: May want to consider Unicode level
+      builder.buildAssert { (input, pos, bounds) in
+        // TODO: How should we handle bounds?
+        CharacterClass.word.isBoundary(
+          input, at: pos, bounds: bounds)
+      }
+
+    case .notWordBoundary:
+      // TODO: May want to consider Unicode level
+      builder.buildAssert { (input, pos, bounds) in
+        // TODO: How should we handle bounds?
+        !CharacterClass.word.isBoundary(
+          input, at: pos, bounds: bounds)
+      }
+    }
+  }
+
   func emitLookaround(
     _ kind: (forwards: Bool, positive: Bool),
     _ child: AST