Clean up, test mid-scalar substring boundaries

Author: natecook1000

Sources/_StringProcessing/Engine/MEBuiltins.swift

@@ -120,7 +120,27 @@ extension Processor {
 
 // MARK: Matching `.`
 extension String {
+  /// Returns the character at `pos`, bounded by `end`, as well as the upper
+  /// boundary of the returned character.
+  ///
+  /// This function handles loading a character from a string while respecting
+  /// an end boundary, even if that end boundary is sub-character or sub-scalar.
+  ///
+  /// - Parameters:
+  ///   - pos: The position to load a character from.
+  ///   - end: The limit for the character at `pos`.
+  /// - Returns:
+  ///   - If `pos` is at or past `end`, this function returns `nil`.
+  ///   - If `end` is between `pos` and the next grapheme cluster boundary (i.e.,
+  ///     `end` is before `self.index(after: pos)`, then the returned character
+  ///     is smaller than the one that would be produced by `self[pos]` and the
+  ///     returned index is at the end of that character.
+  ///   - If `end` is between `pos` and the next grapheme cluster boundary, and
+  ///     is not on a Unicode scalar boundary, the partial scalar is dropped. This
+  ///     can result in a `nil` return or a character that includes only part of
+  ///     the `self[pos]` character.
   func characterAndEnd(at pos: String.Index, limitedBy end: String.Index) -> (Character, String.Index)? {
+    // FIXME: Sink into the stdlib to avoid multiple boundary calculations
     guard pos < end else { return nil }
     let next = index(pos, offsetBy: 1, limitedBy: end) ?? end
     return self[pos..<next].first.map { ($0, next) }

Sources/_StringProcessing/Unicode/WordBreaking.swift

@@ -61,7 +61,8 @@ extension String {
     guard i != range.lowerBound, i != range.upperBound else {
       return true
     }
-    
+    assert(range.contains(i))
+
     // If our index is already in our cache, then this is obviously on a
     // boundary.
     if let cache = cache, cache.contains(i) {

Tests/RegexTests/MatchTests.swift

@@ -79,10 +79,18 @@ func _firstMatch(
   try validateSubstring("A\(input)Z".dropFirst().dropLast())
   do {
     // Test sub-character slicing
-    // TODO: Test sub-scalar slicing
-    let substr = input + "\n"
-    let prevIndex = substr.unicodeScalars.index(substr.endIndex, offsetBy: -1)
-    try validateSubstring(substr[..<prevIndex])
+    let str = input + "\n"
+    let prevIndex = str.unicodeScalars.index(str.endIndex, offsetBy: -1)
+    try validateSubstring(str[..<prevIndex])
+  }
+  do {
+    // Validate that we don't crash when sub-scalar slicing is used
+    // Actual matching behavior is untested here
+    let str = "\u{e9}\(input)e\u{e9}"
+    let upper = str.utf8.index(before: str.endIndex)
+    _ = try regex.firstMatch(in: str[..<upper])
+    let lower = str.utf8.index(after: str.startIndex)
+    _ = try regex.firstMatch(in: str[lower...])
   }
 
   if validateOptimizations {