Merge pull request #109 from hamishknight/ast-quota

Author: hamishknight

Sources/_MatchingEngine/Regex/AST/CustomCharClass.swift

@@ -38,6 +38,10 @@ extension AST {
       /// A single character or escape
       case atom(Atom)
 
+      /// A quoted sequence. Inside a custom character class this just means
+      /// the contents should be interpreted literally.
+      case quote(Quote)
+
       /// A binary operator applied to sets of members `abc&&def`
       case setOperation([Member], Located<SetOp>, [Member])
     }
@@ -76,6 +80,7 @@ extension CustomCC.Member {
     case .custom(let c): return c
     case .range(let r): return r
     case .atom(let a): return a
+    case .quote(let q): return q
     case .setOperation(let lhs, let op, let rhs): return (lhs, op, rhs)
     }
   }

Sources/_MatchingEngine/Regex/Parse/LexicalAnalysis.swift

@@ -453,8 +453,8 @@ extension Source {
   ///
   /// TODO: Need to support some escapes
   ///
-  mutating func lexQuote() throws -> Located<String>? {
-    try recordLoc { src in
+  mutating func lexQuote() throws -> AST.Quote? {
+    let str = try recordLoc { src -> String? in
       if src.tryEat(sequence: #"\Q"#) {
         return try src.expectQuoted(endingWith: #"\E"#).value
       }
@@ -463,6 +463,8 @@ extension Source {
       }
       return nil
     }
+    guard let str = str else { return nil }
+    return AST.Quote(str.value, str.location)
   }
 
   /// Try to consume a comment

Sources/_MatchingEngine/Regex/Parse/Parse.swift

@@ -146,7 +146,7 @@ extension Parser {
 
       //     Quote      -> `lexQuote`
       if let quote = try source.lexQuote() {
-        result.append(.quote(.init(quote.value, loc(_start))))
+        result.append(.quote(quote))
         continue
       }
       //     Quantification  -> QuantOperand Quantifier?
@@ -273,6 +273,12 @@ extension Parser {
         continue
       }
 
+      // Quoted sequence.
+      if let quote = try source.lexQuote() {
+        members.append(.quote(quote))
+        continue
+      }
+
       guard let atom = try source.lexAtom(
         isInCustomCharacterClass: true, priorGroupCount: priorGroupCount)
       else { break }

Sources/_MatchingEngine/Regex/Printing/DumpAST.swift

@@ -71,7 +71,7 @@ extension AST.Concatenation {
 }
 
 extension AST.Quote {
-  public var _dumpBase: String { "quote" }
+  public var _dumpBase: String { "quote \"\(literal)\"" }
 }
 
 extension AST.Trivia {
@@ -203,6 +203,7 @@ extension AST.CustomCharacterClass.Member: _ASTPrintable {
     case .custom(let cc): return "\(cc)"
     case .atom(let a): return "\(a)"
     case .range(let r): return "\(r)"
+    case .quote(let q): return "\(q)"
     case .setOperation(let lhs, let op, let rhs):
       return "op \(lhs) \(op.value) \(rhs)"
     }

Sources/_MatchingEngine/Regex/Printing/PrintAsCanonical.swift

@@ -67,8 +67,7 @@ extension PrettyPrinter {
       output(q.kind.value._canonicalBase)
 
     case let .quote(q):
-      // TODO: Is this really what we want?
-      output("\\Q\(q.literal)\\E")
+      output(q._canonicalBase)
 
     case let .trivia(t):
       // TODO: We might want to output comments...
@@ -110,12 +109,21 @@ extension PrettyPrinter {
       output(r.rhs._canonicalBase)
     case .atom(let a):
       output(a._canonicalBase)
+    case .quote(let q):
+      output(q._canonicalBase)
     case .setOperation:
       output("/* TODO: set operation \(self) */")
     }
   }
 }
 
+extension AST.Quote {
+  var _canonicalBase: String {
+    // TODO: Is this really what we want?
+    "\\Q\(literal)\\E"
+  }
+}
+
 extension AST.Group.Kind {
   var _canonicalBase: String {
     switch self {

Sources/_MatchingEngine/Regex/Printing/PrintAsPattern.swift

@@ -167,6 +167,8 @@ extension PrettyPrinter {
       } else {
         print(a._patternBase)
       }
+    case .quote(let q):
+      print("// TODO: quote \(q.literal._quoted) in custom character classes (should we split it?)")
     case .setOperation:
       print("// TODO: Set operation: \(member)")
     }

Sources/_StringProcessing/ASTBuilder.swift

@@ -214,6 +214,9 @@ func charClass(
 func quote(_ s: String) -> AST {
   .quote(.init(s, .fake))
 }
+func quote_m(_ s: String) -> AST.CustomCharacterClass.Member {
+  .quote(.init(s, .fake))
+}
 
 // MARK: - Atoms
 

Sources/_StringProcessing/CharacterClass.swift

@@ -408,6 +408,11 @@ extension AST.CustomCharacterClass {
           } else {
             return nil
           }
+
+        case .quote(let q):
+          // Decompose quoted literal into literal characters.
+          result += q.literal.map { .character($0) }
+
         case .setOperation(let lhs, let op, let rhs):
           // FIXME: CharacterClass wasn't designed for set operations with
           // multiple components in each operand, we should fix that. For now,

Sources/_StringProcessing/Compiler.swift

@@ -477,7 +477,7 @@ class Compiler {
 public func _compileRegex(
   _ regex: String, _ syntax: SyntaxOptions = .traditional
 ) throws -> Executor {
-  let ast = try parse(regex, .traditional)
+  let ast = try parse(regex, syntax)
   let program = try Compiler(ast: ast).emit()
   return Executor(program: program)
 }

Sources/_StringProcessing/ConsumerInterface.swift

@@ -151,6 +151,20 @@ extension AST.CustomCharacterClass.Member {
       }
       return gen
 
+    case .quote(let q):
+      // TODO: Not optimal.
+      let consumers = try q.literal.map {
+        try AST.Atom(.char($0), .fake).generateConsumer(opts)!
+      }
+      return { input, bounds in
+        for consumer in consumers {
+          if let idx = consumer(input, bounds) {
+            return idx
+          }
+        }
+        return nil
+      }
+
     case .setOperation(let lhs, let op, let rhs):
       // TODO: We should probably have a component type
       // instead of a members array... for now we reconstruct

Tests/RegexTests/MatchTests.swift

@@ -592,6 +592,21 @@ extension RegexTests {
       "--+", input: "123---xyz", match: "---")
     firstMatchTest(
       "~~*", input: "123~~~xyz", match: "~~~")
+
+
+    // Quotes in character classes.
+    firstMatchTest(#"[\Qabc\E]"#, input: "QEa", match: "a")
+    firstMatchTest(#"[\Qabc\E]"#, input: "cxx", match: "c")
+    firstMatchTest(#"[\Qabc\E]+"#, input: "cba", match: "cba")
+    firstMatchTest(#"[\Qa-c\E]+"#, input: "a-c", match: "a-c")
+
+    firstMatchTest(#"["a-c"]+"#, input: "abc", match: "a",
+                   syntax: .experimental)
+    firstMatchTest(#"["abc"]+"#, input: "cba", match: "cba",
+                   syntax: .experimental)
+    firstMatchTest(#"["abc"]+"#, input: #""abc""#, match: "abc",
+                   syntax: .experimental)
+    firstMatchTest(#"["abc"]+"#, input: #""abc""#, match: #""abc""#)
   }
 
   func testCharacterProperties() {

Tests/RegexTests/ParseTests.swift

@@ -433,6 +433,16 @@ extension RegexTests {
               syntax: .experimental)
     parseTest(#""\"""#, quote("\""), syntax: .experimental)
 
+    // Quotes in character classes.
+    parseTest(#"[\Q-\E]"#, charClass(quote_m("-")))
+    parseTest(#"[\Qa-b[[*+\\E]"#, charClass(quote_m(#"a-b[[*+\"#)))
+
+    parseTest(#"["-"]"#, charClass(quote_m("-")), syntax: .experimental)
+    parseTest(#"["a-b[[*+\""]"#, charClass(quote_m(#"a-b[[*+""#)),
+              syntax: .experimental)
+
+    parseTest(#"["-"]"#, charClass(range_m("\"", "\"")))
+
     // MARK: Comments
 
     parseTest(
@@ -979,6 +989,9 @@ extension RegexTests {
     parseNotEqualTest(#"(?+1)"#, #"(?1)"#)
     parseNotEqualTest(#"(?&a)"#, #"(?&b)"#)
 
+    parseNotEqualTest(#"\Qabc\E"#, #"\Qdef\E"#)
+    parseNotEqualTest(#""abc""#, #""def""#)
+
     // TODO: failure tests
   }