Migrate build script to Swift

SwiftSyntaxDevUtils/Package.swift

@@ -0,0 +1,33 @@
+// swift-tools-version:5.7
+
+import PackageDescription
+import Foundation
+
+let package = Package(
+  name: "swift-syntax-dev-utils",
+  platforms: [
+    .macOS(.v10_15)
+  ],
+  products: [
+    .executable(name: "swift-syntax-dev-utils", targets: ["swift-syntax-dev-utils"])
+  ],
+  targets: [
+    .executableTarget(
+      name: "swift-syntax-dev-utils",
+      dependencies: [
+        .product(name: "ArgumentParser", package: "swift-argument-parser")
+      ]
+    )
+  ]
+)
+
+if ProcessInfo.processInfo.environment["SWIFTCI_USE_LOCAL_DEPS"] == nil {
+  // Building standalone.
+  package.dependencies += [
+    .package(url: "https://github.com/apple/swift-argument-parser.git", from: "1.2.2")
+  ]
+} else {
+  package.dependencies += [
+    .package(path: "../../swift-argument-parser")
+  ]
+}

SwiftSyntaxDevUtils/README.md

@@ -0,0 +1,3 @@
+# swift-syntax-dev-utils
+
+Scripts to help build swift-syntax in CI. In most cases, you should not need to run these scripts yourself. The [Contributing Guide](../Contributing.md) contains information of how to build swift-syntax locally.

SwiftSyntaxDevUtils/Sources/swift-syntax-dev-utils/SwiftSyntaxDevUtils.swift

@@ -0,0 +1,34 @@
+//===----------------------------------------------------------------------===//
+//
+// This source file is part of the Swift.org open source project
+//
+// Copyright (c) 2014 - 2023 Apple Inc. and the Swift project authors
+// Licensed under Apache License v2.0 with Runtime Library Exception
+//
+// See https://swift.org/LICENSE.txt for license information
+// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
+//
+//===----------------------------------------------------------------------===//
+
+import ArgumentParser
+
+@main
+struct SwiftSyntaxDevUtils: ParsableCommand {
+
+  static var configuration: CommandConfiguration = CommandConfiguration(
+    abstract: """
+      Build and test script for SwiftSyntax.
+
+      The build script can also drive the test suite included in the SwiftSyntax
+      repo. This requires a custom build of the compiler project since it accesses
+      test utilities that are not shipped as part of the toolchains. See the Testing
+      section for arguments that need to be specified for this.
+      """,
+    subcommands: [
+      Build.self,
+      GenerateSourceCode.self,
+      Test.self,
+      VerifySourceCode.self,
+    ]
+  )
+}

SwiftSyntaxDevUtils/Sources/swift-syntax-dev-utils/commands/Build.swift

@@ -0,0 +1,24 @@
+//===----------------------------------------------------------------------===//
+//
+// This source file is part of the Swift.org open source project
+//
+// Copyright (c) 2014 - 2023 Apple Inc. and the Swift project authors
+// Licensed under Apache License v2.0 with Runtime Library Exception
+//
+// See https://swift.org/LICENSE.txt for license information
+// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
+//
+//===----------------------------------------------------------------------===//
+
+import ArgumentParser
+import Foundation
+
+struct Build: ParsableCommand, BuildCommand {
+  @OptionGroup
+  var arguments: BuildArguments
+
+  func run() throws {
+    try buildTarget(packageDir: Paths.packageDir, targetName: "SwiftSyntax-all")
+    try buildTarget(packageDir: Paths.examplesDir, targetName: "Examples-all")
+  }
+}

SwiftSyntaxDevUtils/Sources/swift-syntax-dev-utils/commands/GenerateSourceCode.swift

@@ -0,0 +1,23 @@
+//===----------------------------------------------------------------------===//
+//
+// This source file is part of the Swift.org open source project
+//
+// Copyright (c) 2014 - 2023 Apple Inc. and the Swift project authors
+// Licensed under Apache License v2.0 with Runtime Library Exception
+//
+// See https://swift.org/LICENSE.txt for license information
+// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
+//
+//===----------------------------------------------------------------------===//
+
+import ArgumentParser
+import Foundation
+
+struct GenerateSourceCode: ParsableCommand, SourceCodeGeneratorCommand {
+  @OptionGroup
+  var arguments: SourceCodeGeneratorArguments
+
+  func run() throws {
+    try self.runCodeGeneration(sourceDir: Paths.sourcesDir)
+  }
+}

SwiftSyntaxDevUtils/Sources/swift-syntax-dev-utils/commands/Test.swift

@@ -0,0 +1,160 @@
+//===----------------------------------------------------------------------===//
+//
+// This source file is part of the Swift.org open source project
+//
+// Copyright (c) 2014 - 2023 Apple Inc. and the Swift project authors
+// Licensed under Apache License v2.0 with Runtime Library Exception
+//
+// See https://swift.org/LICENSE.txt for license information
+// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
+//
+//===----------------------------------------------------------------------===//
+
+import ArgumentParser
+import Foundation
+
+struct Test: ParsableCommand, BuildCommand {
+  @OptionGroup
+  var arguments: BuildArguments
+
+  @Flag(help: "Don't run lit-based tests")
+  var skipLitTests: Bool = false
+
+  @Option(
+    help: """
+      Path to the FileCheck executable that was built as part of the LLVM repository.
+      If not specified, it will be looked up from PATH.
+      """
+  )
+  var filecheckExec: String?
+
+  func run() throws {
+    try buildExample(exampleName: "ExamplePlugin")
+
+    try runTests()
+
+    logSection("All tests passed")
+  }
+
+  private func runTests() throws {
+    logSection("Running SwiftSyntax Tests")
+
+    if !skipLitTests {
+      try runLitTests()
+    }
+
+    try runXCTests()
+  }
+
+  private func runLitTests() throws {
+    logSection("Running lit-based tests")
+
+    guard FileManager.default.fileExists(atPath: Paths.litExec.path) else {
+      throw ScriptExectutionError(
+        message: """
+          Error: Could not find lit.py.
+          Looking at '\(Paths.litExec.path)'.
+
+          Make sure you have the llvm repo checked out next to the swift-syntax repo.
+          Refer to README.md for more information.
+          """
+      )
+    }
+
+    let examplesBinPath = try findExamplesBinPath()
+
+    var litCall = [
+      Paths.litExec.path,
+      Paths.packageDir.appendingPathComponent("lit_tests").path,
+    ]
+
+    if let filecheckExec {
+      litCall += ["--param", "FILECHECK=" + filecheckExec]
+    }
+
+    litCall += ["--param", "EXAMPLES_BIN_PATH=" + examplesBinPath.path]
+    litCall += ["--param", "TOOLCHAIN=" + arguments.toolchain.path]
+
+    // Print all failures
+    litCall += ["--verbose"]
+    // Don't show all commands if verbose is not enabled
+    if !arguments.verbose {
+      litCall += ["--succinct"]
+    }
+
+    guard let pythonExec = Paths.python3Exec else {
+      throw ScriptExectutionError(message: "Didn't find python3 executable")
+    }
+
+    let process = ProcessRunner(
+      executableURL: pythonExec,
+      arguments: litCall
+    )
+
+    let processOutput = try process.run(verbose: arguments.verbose)
+
+    if !processOutput.stdout.isEmpty {
+      logSection("lit test stdout")
+      print(processOutput.stdout)
+    }
+
+    if !processOutput.stderr.isEmpty {
+      logSection("lit test stderr")
+      print(processOutput.stderr)
+    }
+  }
+
+  private func runXCTests() throws {
+    logSection("Running XCTests")
+    var swiftpmCallArguments: [String] = []
+
+    if arguments.verbose {
+      swiftpmCallArguments += ["--verbose"]
+    }
+
+    swiftpmCallArguments += ["--test-product", "swift-syntaxPackageTests"]
+
+    var additionalEnvironment: [String: String] = [:]
+    additionalEnvironment["SWIFT_BUILD_SCRIPT_ENVIRONMENT"] = "1"
+
+    if arguments.enableRawSyntaxValidation {
+      additionalEnvironment["SWIFTSYNTAX_ENABLE_RAWSYNTAX_VALIDATION"] = "1"
+    }
+
+    if arguments.enableTestFuzzing {
+      additionalEnvironment["SWIFTPARSER_ENABLE_ALTERNATE_TOKEN_INTROSPECTION"] = "1"
+    }
+
+    // Tell other projects in the unified build to use local dependencies
+    additionalEnvironment["SWIFTCI_USE_LOCAL_DEPS"] = "1"
+    additionalEnvironment["SWIFT_SYNTAX_PARSER_LIB_SEARCH_PATH"] =
+      arguments.toolchain
+      .appendingPathComponent("lib")
+      .appendingPathComponent("swift")
+      .appendingPathComponent("macosx")
+      .path
+
+    try invokeSwiftPM(
+      action: "test",
+      packageDir: Paths.packageDir,
+      additionalArguments: swiftpmCallArguments,
+      additionalEnvironment: additionalEnvironment
+    )
+  }
+
+  private func findSwiftpmBinPath(packageDir: URL) throws -> String {
+    return try invokeSwiftPM(
+      action: "build",
+      packageDir: packageDir,
+      additionalArguments: ["--show-bin-path"],
+      additionalEnvironment: [:]
+    )
+  }
+
+  /// This returns a path to the build examples folder.
+  /// Example: '<workingDir>/swift-syntax/Examples/.build/arm64-apple-macosx/debug
+  private func findExamplesBinPath() throws -> URL {
+    let stdOut = try findSwiftpmBinPath(packageDir: Paths.examplesDir)
+    return URL(fileURLWithPath: stdOut.trimmingCharacters(in: .whitespacesAndNewlines))
+  }
+}

SwiftSyntaxDevUtils/Sources/swift-syntax-dev-utils/commands/VerifySourceCode.swift

@@ -0,0 +1,67 @@
+//===----------------------------------------------------------------------===//
+//
+// This source file is part of the Swift.org open source project
+//
+// Copyright (c) 2014 - 2023 Apple Inc. and the Swift project authors
+// Licensed under Apache License v2.0 with Runtime Library Exception
+//
+// See https://swift.org/LICENSE.txt for license information
+// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
+//
+//===----------------------------------------------------------------------===//
+
+import ArgumentParser
+import Foundation
+
+fileprivate var modules: [String] {
+  ["SwiftParser", "SwiftParserDiagnostics", "SwiftSyntax", "SwiftSyntaxBuilder"]
+}
+
+struct VerifySourceCode: ParsableCommand, SourceCodeGeneratorCommand {
+  @OptionGroup
+  var arguments: SourceCodeGeneratorArguments
+
+  func run() throws {
+    let tempDir = FileManager.default.temporaryDirectory
+
+    try self.runCodeGeneration(sourceDir: tempDir)
+
+    logSection("Verifing code generated files")
+
+    guard let diffExec = Paths.diffExec else {
+      throw ScriptExectutionError(message: "Didn't find a diff execution path")
+    }
+
+    for module in modules {
+      let selfGeneratedDir = tempDir.appendingPathComponent(module).appendingPathComponent("generated")
+      let userGeneratedDir = Paths.sourcesDir.appendingPathComponent(module).appendingPathComponent("generated")
+
+      let process = ProcessRunner(
+        executableURL: diffExec,
+        arguments: [
+          "--recursive",
+          "--exclude",
+          ".*",  // Exclude dot files like .DS_Store
+          "--context=0",
+          selfGeneratedDir.path,
+          userGeneratedDir.path,
+        ]
+      )
+
+      let result = try process.run(verbose: arguments.verbose)
+
+      if !result.stderr.isEmpty {
+        throw ScriptExectutionError(
+          message: """
+            FAIL: code-generated files committed to repository do
+            not match generated ones. Please re-generate the
+            code-generated-files using the following command, open a PR to the
+            SwiftSyntax project and merge it alongside the main PR.
+            $ swift run swift-syntax-dev-utils generate-source-code
+            /path/to/toolchain.xctoolchain/usr
+            """
+        )
+      }
+    }
+  }
+}