swiftlang · slavapestov · Mar 18, 2022 · Jan 23, 2022 · Feb 17, 2022 · Feb 17, 2022
diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -0,0 +1,17 @@
+
+cmake_minimum_required(VERSION 3.18)
+project(SwiftExperimentalStringProcessing
+  LANGUAGES Swift)
+
+if(CMAKE_SYSTEM_NAME STREQUAL Windows OR CMAKE_SYSTEM_NAME STREQUAL Darwin)
+  option(BUILD_SHARED_LIBS "Build shared libraries by default" YES)
+endif()
+
+set(CMAKE_ARCHIVE_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/lib)
+set(CMAKE_LIBRARY_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/lib)
+set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/bin)
+set(CMAKE_Swift_MODULE_DIRECTORY ${CMAKE_BINARY_DIR}/swift)
+
+find_package(ArgumentParser CONFIG)
+
+add_subdirectory(Sources)
diff --git a/Documentation/Evolution/RegexSyntax.md b/Documentation/Evolution/RegexSyntax.md
diff --git a/README.md b/README.md
@@ -9,3 +9,65 @@ See [Declarative String Processing Overview][decl-string]
 ## Requirements
 
 - [Swift Trunk Development Snapshot](https://www.swift.org/download/#snapshots) DEVELOPMENT-SNAPSHOT-2022-02-03 or later.
+
+## Integration with Swift
+
+`_MatchingEngine`, `_CUnicode` and `_StringProcessing` are specially integrated modules that are built as part of apple/swift.
+
+Specifically, `_MatchingEngine` contains the parser for regular expression literals and is built both as part of the compiler and as a core library. `_CUnicode` and `_StringProcessing` are built together as a core library named `_StringProcessing`.
+
+| Module              | Swift toolchain component                                                            |
+| ------------------- | ------------------------------------------------------------------------------------ |
+| `_MatchingEngine`   | `SwiftCompilerSources/Sources/ExperimentalRegex` and `stdlib/public/_MatchingEngine` |
+| `_CUnicode`         | `stdlib/public/_StringProcessing`                                                    |
+| `_StringProcessing` | `stdlib/public/_StringProcessing`                                                    |
+
+### Branching scheme
+
+#### Development branch
+
+The `main` branch is the branch for day-to-day development. Generally, you should create PRs against this branch.
+
+#### Swift integration branches
+
+Branches whose name starts with `swift/` are Swift integration branches similar to those in [apple/llvm-project](https://github.com/apple/llvm-project). For each branch, dropping the `swift/` prefix is the corresponding branch in [apple/swift](https://github.com/apple/swift).
+
+| apple/swift branch  | apple/swift-experimental-string-processing branch     |
+| ------------------- | ----------------------------------------------------- |
+| main                | swift/main                                            |
+| release/5.7         | swift/release/5.7                                     |
+| ...                 | swift/...                                             |
+
+A pair of corresponding branches are expected to build successfully together and pass all tests.
+
+### Integration workflow
+
+To integrate the latest changes in apple/swift-experimental-string-processing to apple/swift, carefully follow the workflow: 
+
+- Create pull requests.
+  - Create a pull request in apple/swift-experimental-string-processing from `main` to `swift/main`, e.g. "[Integration] main -> swift/main".
+  - If apple/swift needs to be modified to work with the latest `main` in apple/swift-experimental-string-processing, create a pull request in apple/swift.
+- Trigger CI.
+  - In the apple/swift-experimental-string-processing pull request, trigger CI using the following command (replacing `<PR NUMBER>` with the apple/swift pull request number, if any):
+    ```
+    apple/swift#<PR NUMBER> # use this line only if there is an corresponding apple/swift PR
+    @swift-ci please test
+    ```
+  - In the apple/swift pull request (if any), trigger CI using the following command (replacing `<PR NUMBER>` with the apple/swift-experimental-string-processing pull request number):
+    ```
+    apple/swift-experimental-string-processing#<PR NUMBER>
+    @swift-ci please test
+    ```
+- Merge when approved.
+  - Merge the pull request in apple/swift-experimental-string-processing as a **merge commit**.
+  - Merge the pull request in apple/swift (if any).
+
+### Development notes
+
+Compiler integration can be tricky. Use special caution when developing `_MatchingEngine`, `_CUnicode` and `_StringProcessing` modules.
+
+- Do not change the names of these modules without due approval from compiler and infrastructure teams.
+- Do not modify the existing ABI (e.g. C API, serialization format) between the regular expression parser and the Swift compiler unless absolutely necessary. 
+- Always minimize the number of lockstep integrations, i.e. when apple/swift-experimental-string-processing and apple/swift have to change together. Whenever possible, introduce new API first, migrate Swift compiler onto it, and then deprecate old API. Use versioning if helpful.
+- In `_StringProcessing`, do not write fully qualified references to symbols in `_CUnicode`, and always wrap `import _CUnicode` in a `#if canImport(_CUnicode)`. This is because `_CUnicode` is built as part of `_StringProcessing` with CMake.
+- In `_MatchingEngine`, do not write fully qualified references to `_MatchingEngine` itself. This is because `_MatchingEngine` is built as `ExperimentalRegex` in `SwiftCompilerSources/` with CMake. 
diff --git a/Sources/CMakeLists.txt b/Sources/CMakeLists.txt
@@ -0,0 +1,6 @@
+
+add_subdirectory(_Unicode)
+add_subdirectory(_MatchingEngine)
+add_subdirectory(_StringProcessing)
+add_subdirectory(Prototypes)
+add_subdirectory(VariadicsGenerator)
diff --git a/Sources/Prototypes/CMakeLists.txt b/Sources/Prototypes/CMakeLists.txt
@@ -0,0 +1,18 @@
+
+add_library(Prototypes
+  Combinators/Combinators.swift
+  PEG/PEG.swift
+  PEG/PEGCode.swift
+  PEG/PEGCompile.swift
+  PEG/PEGCore.swift
+  PEG/PEGInterpreter.swift
+  PEG/PEGTranspile.swift
+  PEG/PEGVM.swift
+  PEG/PEGVMExecute.swift
+  PEG/Printing.swift
+  PTCaRet/Interpreter.swift
+  PTCaRet/PTCaRet.swift
+  TourOfTypes/CharacterClass.swift
+  TourOfTypes/Literal.swift)
+target_link_libraries(Prototypes PUBLIC
+  _MatchingEngine)
diff --git a/Sources/Prototypes/PEG/PEGCode.swift b/Sources/Prototypes/PEG/PEGCode.swift
@@ -9,7 +9,7 @@
 //
 //===----------------------------------------------------------------------===//
 
-import _StringProcessing
+@testable import _StringProcessing
 
 extension PEG.VM {
   struct Code {

diff --git a/Sources/Prototypes/PEG/PEGCompile.swift b/Sources/Prototypes/PEG/PEGCompile.swift
@@ -9,7 +9,7 @@
 //
 //===----------------------------------------------------------------------===//
 
-import _StringProcessing
+@testable import _StringProcessing
 
 extension PEG.VM {
   typealias InIndex = Input.Index

diff --git a/Sources/Prototypes/PEG/PEGCore.swift b/Sources/Prototypes/PEG/PEGCore.swift
@@ -9,7 +9,7 @@
 //
 //===----------------------------------------------------------------------===//
 
-import _StringProcessing
+@testable import _StringProcessing
 let emitComments = true
 
 struct PEGCore<

diff --git a/Sources/Prototypes/PEG/PEGTranspile.swift b/Sources/Prototypes/PEG/PEGTranspile.swift
@@ -9,8 +9,7 @@
 //
 //===----------------------------------------------------------------------===//
 
-import _MatchingEngine
-import _StringProcessing
+@testable import _StringProcessing
 
 extension PEG.VM where Input == String {
   typealias MEProg = MEProgram<String>

diff --git a/Sources/Prototypes/PEG/PEGVM.swift b/Sources/Prototypes/PEG/PEGVM.swift
@@ -9,7 +9,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-import _StringProcessing
+
+@testable import _StringProcessing
 
 extension PEG {
 

diff --git a/Sources/Prototypes/PEG/Printing.swift b/Sources/Prototypes/PEG/Printing.swift
@@ -9,7 +9,7 @@
 //
 //===----------------------------------------------------------------------===//
 
-import _StringProcessing
+@testable import _StringProcessing
 
 extension PEGCore.Instruction: InstructionProtocol {
   var operandPC: InstructionAddress? { self.pc }

diff --git a/Sources/VariadicsGenerator/CMakeLists.txt b/Sources/VariadicsGenerator/CMakeLists.txt
@@ -0,0 +1,7 @@
+
+add_executable(VariadicsGenerator
+  VariadicsGenerator.swift)
+target_compile_options(VariadicsGenerator PRIVATE
+  -parse-as-library)
+target_link_libraries(VariadicsGenerator PUBLIC
+  ArgumentParser)
diff --git a/Sources/_MatchingEngine/CMakeLists.txt b/Sources/_MatchingEngine/CMakeLists.txt
@@ -0,0 +1,46 @@
+
+add_library(_MatchingEngine
+  Engine/Backtracking.swift
+  Engine/Builder.swift
+  Engine/Capture.swift
+  Engine/Consume.swift
+  Engine/Engine.swift
+  Engine/InstPayload.swift
+  Engine/Instruction.swift
+  Engine/Processor.swift
+  Engine/Program.swift
+  Engine/Registers.swift
+  Engine/Tracing.swift
+  Regex/AST/AST.swift
+  Regex/AST/ASTAction.swift
+  Regex/AST/ASTProtocols.swift
+  Regex/AST/Atom.swift
+  Regex/AST/Conditional.swift
+  Regex/AST/CustomCharClass.swift
+  Regex/AST/Group.swift
+  Regex/AST/MatchingOptions.swift
+  Regex/AST/Quantification.swift
+  Regex/Parse/CaptureStructure.swift
+  Regex/Parse/CharacterPropertyClassification.swift
+  Regex/Parse/Diagnostics.swift
+  Regex/Parse/LexicalAnalysis.swift
+  Regex/Parse/Mocking.swift
+  Regex/Parse/Parse.swift
+  Regex/Parse/Source.swift
+  Regex/Parse/SourceLocation.swift
+  Regex/Parse/SyntaxOptions.swift
+  Regex/Printing/DumpAST.swift
+  Regex/Printing/PrettyPrinter.swift
+  Regex/Printing/PrintAsCanonical.swift
+  Regex/Printing/PrintAsPattern.swift
+  Regex/Printing/RenderRanges.swift
+  Utility/AllScalars.swift
+  Utility/Formatting.swift
+  Utility/Misc.swift
+  Utility/MissingUnicode.swift
+  Utility/Protocols.swift
+  Utility/TypeConstruction.swift
+  Utility/TypedIndex.swift
+  Utility/TypedInt.swift)
+target_compile_options(_MatchingEngine PRIVATE
+  -enable-library-evolution)
diff --git a/Sources/_MatchingEngine/Regex/Parse/CharacterPropertyClassification.swift b/Sources/_MatchingEngine/Regex/Parse/CharacterPropertyClassification.swift
@@ -381,7 +381,7 @@ extension Source {
       return .generalCategory(cat)
     }
     if let script = classifyScriptProperty(value) {
-      return .script(script)
+      return .scriptExtension(script)
     }
     if let posix = classifyPOSIX(value) {
       return .posix(posix)