Migrate build script to Swift

Author: kimdv

CodeGeneration/Package.swift renamed to SwiftSyntaxCLI/Package.swift

@@ -1,22 +1,22 @@
-// swift-tools-version:5.7
+// swift-tools-version:5.8
 
 import PackageDescription
 import Foundation
 
 let package = Package(
-  name: "CodeGeneration",
+  name: "SwiftSyntaxCLI",
   platforms: [
     .macOS(.v10_15)
   ],
   products: [
-    .executable(name: "generate-swiftsyntax", targets: ["generate-swiftsyntax"])
+    .executable(name: "swift-syntax-cli", targets: ["swift-syntax-cli"])
   ],
   dependencies: [
-    .package(url: "..", revision: "HEAD")
+    .package(path: "..")
   ],
   targets: [
     .executableTarget(
-      name: "generate-swiftsyntax",
+      name: "swift-syntax-cli",
       dependencies: [
         .product(name: "SwiftSyntax", package: "swift-syntax"),
         .product(name: "SwiftSyntaxBuilder", package: "swift-syntax"),
@@ -45,7 +45,7 @@ let package = Package(
 if ProcessInfo.processInfo.environment["SWIFTCI_USE_LOCAL_DEPS"] == nil {
   // Building standalone.
   package.dependencies += [
-    .package(url: "https://github.com/apple/swift-argument-parser.git", from: "1.2.2")
+    .package(url: "https://github.com/apple/swift-argument-parser.git", .upToNextMinor(from: "1.2.2"))
   ]
 } else {
   package.dependencies += [

CodeGeneration/README.md renamed to SwiftSyntaxCLI/README.md

@@ -1,20 +1,21 @@
-# Code-Generation for SwiftSyntax
+# SwiftSyntaxCLI
+
+## Code-Generation for SwiftSyntax
 
 This directory contains file to generate source code that is part of the SwiftSyntax package. If you are looking to generate Swift code yourself, you might be interested in [SwiftSyntaxBuilder](../Sources/SwiftSyntaxBuilder).
 
 Some source code inside SwiftSyntax is generated using [SwiftSyntaxBuilder](../Sources/SwiftSyntaxBuilder), a Swift library whose purpose is to generate Swift code using Swift itself. This kind of code generation is performed by the Swift package defined in this directory.
 
-This directory is a standalone package that uses HEAD of the current branch. This guarantees that when `generate-swiftsyntax` is run, it can't break its own build when run multiple times without committing. 
-This means that `CodeGeneration` will build against your latest local commit of SwiftSyntax. If you are making changes to `SwiftSyntax` that affect how code is being generated, commit your SwiftSyntax changes (pushing is not necessary) and re-generate files afterwards.
+This directory is a standalone package that uses HEAD of the current branch. This guarantees that when `generate-source-code` is run, it can't break its own build when run multiple times without committing. 
+This means that `SwiftSyntaxCLI` will build against your latest local commit of SwiftSyntax. If you are making changes to `SwiftSyntax` that affect how code is being generated, commit your SwiftSyntax changes (pushing is not necessary) and re-generate files afterwards.
 
-To re-generate the files after changing `CodeGeneration` run the `generate-swiftsyntax` 
-target of `CodeGeneration` and pass `path/to/swift-syntax/Sources` as the argument.
+To re-generate the files after changing `SwiftSyntaxCLI` run the `generate-source-code` 
+target of `SwiftSyntaxCLI` and pass `path/to/swift-syntax/Sources` as the argument.
 
 On the command line, this would be
 ```bash
-swift run --package-path CodeGeneration generate-swiftsyntax Sources
+swift run --package-path SwiftSyntaxCLI generate-source-code Sources
 ```
 
-Or if you open the `CodeGeneration` package in Xcode, you can add the 
-`generate-swift syntax ../Sources` arguments using Product -> Scheme -> Edit Scheme…
-
+Or if you open the `SwiftSyntaxCLI` package in Xcode, you can add the 
+`generate-source-code ../Sources` arguments using Product -> Scheme -> Edit Scheme…

CodeGeneration/Sources/SyntaxSupport/AttributeKinds.swift renamed to SwiftSyntaxCLI/Sources/SyntaxSupport/AttributeKinds.swift

@@ -0,0 +1,34 @@
+//===----------------------------------------------------------------------===//
+//
+// This source file is part of the Swift.org open source project
+//
+// Copyright (c) 2014 - 2023 Apple Inc. and the Swift project authors
+// Licensed under Apache License v2.0 with Runtime Library Exception
+//
+// See https://swift.org/LICENSE.txt for license information
+// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
+//
+//===----------------------------------------------------------------------===//
+
+import ArgumentParser
+
+@main
+struct SwiftSyntaxCLI: ParsableCommand {
+
+  static var configuration: CommandConfiguration = CommandConfiguration(
+    abstract: """
+      Build and test script for SwiftSyntax.
+
+      The build script can also drive the test suite included in the SwiftSyntax
+      repo. This requires a custom build of the compiler project since it accesses
+      test utilities that are not shipped as part of the toolchains. See the Testing
+      section for arguments that need to be specified for this.
+      """,
+    subcommands: [
+      Build.self,
+      GenerateSourceCode.self,
+      Test.self,
+      VerifySourceCode.self,
+    ]
+  )
+}

CodeGeneration/Sources/SyntaxSupport/AttributeNodes.swift renamed to SwiftSyntaxCLI/Sources/SyntaxSupport/AttributeNodes.swift

@@ -0,0 +1,30 @@
+//===----------------------------------------------------------------------===//
+//
+// This source file is part of the Swift.org open source project
+//
+// Copyright (c) 2014 - 2023 Apple Inc. and the Swift project authors
+// Licensed under Apache License v2.0 with Runtime Library Exception
+//
+// See https://swift.org/LICENSE.txt for license information
+// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
+//
+//===----------------------------------------------------------------------===//
+
+import ArgumentParser
+import Foundation
+
+struct Build: ParsableCommand, Builder {
+  @OptionGroup
+  var arguments: BuilderArguments
+
+  func run() throws {
+    // Until rdar://53881101 is implemented, we cannot request a build of multiple
+    // targets simultaneously. For now, just build one product after the other.
+    try buildProduct(productName: "SwiftSyntax")
+    try buildProduct(productName: "SwiftSyntaxBuilder")
+
+    // Build examples
+    try buildExample(exampleName: "AddOneToIntegerLiterals")
+    try buildExample(exampleName: "CodeGenerationUsingSwiftSyntaxBuilder")
+  }
+}

CodeGeneration/Sources/SyntaxSupport/AvailabilityNodes.swift renamed to SwiftSyntaxCLI/Sources/SyntaxSupport/AvailabilityNodes.swift

@@ -0,0 +1,23 @@
+//===----------------------------------------------------------------------===//
+//
+// This source file is part of the Swift.org open source project
+//
+// Copyright (c) 2014 - 2023 Apple Inc. and the Swift project authors
+// Licensed under Apache License v2.0 with Runtime Library Exception
+//
+// See https://swift.org/LICENSE.txt for license information
+// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
+//
+//===----------------------------------------------------------------------===//
+
+import ArgumentParser
+import Foundation
+
+struct GenerateSourceCode: ParsableCommand, Generator {
+  @OptionGroup
+  var arguments: GeneratorArguments
+
+  func run() throws {
+    try self.runCodeGeneration(sourceDir: Paths.sourcesDir)
+  }
+}

CodeGeneration/Sources/SyntaxSupport/BuilderInitializableTypes.swift renamed to SwiftSyntaxCLI/Sources/SyntaxSupport/BuilderInitializableTypes.swift

@@ -0,0 +1,169 @@
+//===----------------------------------------------------------------------===//
+//
+// This source file is part of the Swift.org open source project
+//
+// Copyright (c) 2014 - 2023 Apple Inc. and the Swift project authors
+// Licensed under Apache License v2.0 with Runtime Library Exception
+//
+// See https://swift.org/LICENSE.txt for license information
+// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
+//
+//===----------------------------------------------------------------------===//
+
+import ArgumentParser
+import Foundation
+
+struct Test: ParsableCommand, Builder {
+  @OptionGroup
+  var arguments: BuilderArguments
+
+  @Flag(help: "Don't run lit-based tests")
+  var skipLitTests: Bool = false
+
+  @Argument(
+    help: """
+      Path to the FileCheck executable that was built as part of the LLVM repository.
+      If not specified, it will be looked up from PATH.
+      """
+  )
+  var filecheckExec: String?
+
+  func run() throws {
+    try buildProduct(productName: "lit-test-helper")
+    try buildExample(exampleName: "ExamplePlugin")
+
+    try runTests()
+
+    print("** All tests passed **")
+  }
+
+  private func runTests() throws {
+    print("** Running SwiftSyntax Tests **")
+
+    if !skipLitTests {
+      try runLitTests()
+    }
+
+    try runXctests()
+  }
+
+  private func runLitTests() throws {
+    print("** Running lit-based tests **")
+
+    guard FileManager.default.fileExists(atPath: Paths.litExec.relativePath) else {
+      fatalError(
+        """
+        Error: Could not find lit.py.
+        Looking at '\(Paths.litExec.relativePath)'.
+
+        Make sure you have the llvm repo checked out next to the swift-syntax repo.
+        Refer to README.md for more information.
+        """
+      )
+    }
+
+    guard FileManager.default.fileExists(atPath: Paths.incrTransferRoundtripExec.relativePath) else {
+      fatalError(
+        """
+        Error: Could not find incr_transfer_round_trip.py.
+
+        Make sure you have the main swift repo checked out next to the swift-syntax
+        repo.
+        Refer to README.md for more information.
+        """
+      )
+    }
+
+    let productBinPath = try findProductBinPath()
+    let examplesBinPath = try findExamplesBinPath()
+
+    let litTestHelperExec = productBinPath + "/lit-test-helper"
+    var litCall = [
+      Paths.litExec.relativePath,
+      Paths.packageDir.relativePath,
+      "lit_tests",
+    ]
+
+    if let filecheckExec {
+      litCall += ["--param", "FILECHECK=" + filecheckExec]
+    }
+
+    litCall += ["--param", "LIT_TEST_HELPER=" + litTestHelperExec]
+    litCall += ["--param", "INCR_TRANSFER_ROUND_TRIP.PY=" + Paths.incrTransferRoundtripExec.relativePath]
+
+    litCall += ["--param", "EXAMPLES_BIN_PATH=" + examplesBinPath]
+    litCall += ["--param", "TOOLCHAIN=" + arguments.toolchain]
+
+    // Print all failures
+    litCall += ["--verbose"]
+    // Don't show all commands if verbose is not enabled
+    if !arguments.verbose {
+      litCall += ["--succinct"]
+    }
+
+    try runPython(arguments: litCall)
+  }
+
+  private func runXctests() throws {
+    print("** Running XCTests **")
+    var swiftpmCallArguments: [String] = []
+
+    if arguments.verbose {
+      swiftpmCallArguments += ["--verbose"]
+    }
+
+    swiftpmCallArguments += ["--test-product", "SwiftSyntaxPackageTests"]
+
+    var env = ProcessInfo.processInfo.environment
+    env["SWIFT_BUILD_SCRIPT_ENVIRONMENT"] = "1"
+
+    if arguments.enableRawsyntaxValidation {
+      env["SWIFTSYNTAX_ENABLE_RAWSYNTAX_VALIDATION"] = "1"
+    }
+
+    if arguments.enableTestFuzzing {
+      env["SWIFTPARSER_ENABLE_ALTERNATE_TOKEN_INTROSPECTION"] = "1"
+    }
+
+    // Tell other projects in the unified build to use local dependencies
+    env["SWIFTCI_USE_LOCAL_DEPS"] = "1"
+    env["SWIFT_SYNTAX_PARSER_LIB_SEARCH_PATH"] =
+      URL(fileURLWithPath: arguments.toolchain)
+      .appendingPathComponent("lib")
+      .appendingPathComponent("swift")
+      .appendingPathComponent("macosx")
+      .relativePath
+
+    try swiftpmInvocation(
+      action: "test",
+      packageDir: Paths.packageDir,
+      additionalArguments: swiftpmCallArguments,
+      env: env
+    )
+  }
+
+  private func findSwiftpmBinPath(packageDir: URL) throws -> String {
+    return try swiftpmInvocation(
+      action: "build",
+      packageDir: packageDir,
+      additionalArguments: ["--show-bin-path"],
+      env: [:]
+    )
+  }
+
+  private func findProductBinPath() throws -> String {
+    return try findSwiftpmBinPath(packageDir: Paths.packageDir)
+  }
+
+  private func findExamplesBinPath() throws -> String {
+    return try findSwiftpmBinPath(packageDir: Paths.examplesDir)
+  }
+
+  private func runPython(arguments: [String]) throws {
+    let process = Process()
+    process.executableURL = URL(fileURLWithPath: "/usr/bin/python3")
+    process.arguments = arguments
+    try process.run()
+    process.waitUntilExit()
+  }
+}

CodeGeneration/Sources/SyntaxSupport/Child.swift renamed to SwiftSyntaxCLI/Sources/SyntaxSupport/Child.swift

@@ -0,0 +1,62 @@
+//===----------------------------------------------------------------------===//
+//
+// This source file is part of the Swift.org open source project
+//
+// Copyright (c) 2014 - 2023 Apple Inc. and the Swift project authors
+// Licensed under Apache License v2.0 with Runtime Library Exception
+//
+// See https://swift.org/LICENSE.txt for license information
+// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
+//
+//===----------------------------------------------------------------------===//
+
+import ArgumentParser
+import Foundation
+
+struct VerifySourceCode: ParsableCommand, Generator {
+  @OptionGroup
+  var arguments: GeneratorArguments
+
+  func run() throws {
+    let tempDir = FileManager.default.temporaryDirectory
+
+    try self.runCodeGeneration(sourceDir: tempDir)
+
+    print("** Verifing code generated files **")
+
+    for module in Paths.modules {
+      let selfGeneratedDir = tempDir.appendingPathComponent(module).appendingPathComponent("generated")
+      let userGeneratedDir = Paths.sourcesDir.appendingPathComponent(module).appendingPathComponent("generated")
+      try checkGeneratedFilesMatch(selfGeneratedDir: selfGeneratedDir, userGeneratedDir: userGeneratedDir)
+    }
+  }
+
+  private func checkGeneratedFilesMatch(
+    selfGeneratedDir: URL,
+    userGeneratedDir: URL
+  ) throws {
+
+    let selfGeneratedFiles = try FileManager.default.contentsOfDirectory(at: selfGeneratedDir, includingPropertiesForKeys: nil)
+    let userGeneratedFiles = try FileManager.default.contentsOfDirectory(at: userGeneratedDir, includingPropertiesForKeys: nil)
+
+    zip(selfGeneratedFiles, userGeneratedFiles)
+      .forEach {
+        if arguments.verbose {
+          print("Comparing \($0.relativePath) with \($1.relativePath) ...")
+        }
+
+        if !FileManager.default.contentsEqual(atPath: $0.relativePath, andPath: $1.relativePath) {
+          fatalError(
+            """
+            FAIL: code-generated files committed to repository do
+            not match generated ones. Please re-generate the
+            code-generated-files using the following command, open a PR to the
+            SwiftSyntax project and merge it alongside the main PR.
+            $ swift-syntax/build-script.py generate-source-code
+            --toolchain /path/to/toolchain.xctoolchain/usr
+            """
+          )
+        }
+      }
+  }
+}