From c148db3532771bdbfe3f8e5e05de1f4348d8f9c0 Mon Sep 17 00:00:00 2001 From: Kim de Vos Date: Thu, 18 May 2023 14:27:13 +0200 Subject: [PATCH] Migrate build script to Swift --- SwiftSyntaxDevUtils/Package.swift | 33 + SwiftSyntaxDevUtils/README.md | 3 + .../SwiftSyntaxDevUtils.swift | 34 + .../commands/Build.swift | 24 + .../commands/GenerateSourceCode.swift | 23 + .../commands/Test.swift | 160 ++++ .../commands/VerifySourceCode.swift | 67 ++ .../common/BuildArguments.swift | 70 ++ .../common/BuildCommand.swift | 118 +++ .../common/Logger.swift | 31 + .../swift-syntax-dev-utils/common/Paths.swift | 111 +++ .../common/ProcessRunner.swift | 122 +++ .../common/ScriptExectutionError.swift | 21 + .../common/SourceCodeGeneratorArguments.swift | 25 + .../common/SourceCodeGeneratorCommand.swift | 49 ++ build-script.py | 721 ------------------ 16 files changed, 891 insertions(+), 721 deletions(-) create mode 100644 SwiftSyntaxDevUtils/Package.swift create mode 100644 SwiftSyntaxDevUtils/README.md create mode 100644 SwiftSyntaxDevUtils/Sources/swift-syntax-dev-utils/SwiftSyntaxDevUtils.swift create mode 100644 SwiftSyntaxDevUtils/Sources/swift-syntax-dev-utils/commands/Build.swift create mode 100644 SwiftSyntaxDevUtils/Sources/swift-syntax-dev-utils/commands/GenerateSourceCode.swift create mode 100644 SwiftSyntaxDevUtils/Sources/swift-syntax-dev-utils/commands/Test.swift create mode 100644 SwiftSyntaxDevUtils/Sources/swift-syntax-dev-utils/commands/VerifySourceCode.swift create mode 100644 SwiftSyntaxDevUtils/Sources/swift-syntax-dev-utils/common/BuildArguments.swift create mode 100644 SwiftSyntaxDevUtils/Sources/swift-syntax-dev-utils/common/BuildCommand.swift create mode 100644 SwiftSyntaxDevUtils/Sources/swift-syntax-dev-utils/common/Logger.swift create mode 100644 SwiftSyntaxDevUtils/Sources/swift-syntax-dev-utils/common/Paths.swift create mode 100644 SwiftSyntaxDevUtils/Sources/swift-syntax-dev-utils/common/ProcessRunner.swift create mode 100644 SwiftSyntaxDevUtils/Sources/swift-syntax-dev-utils/common/ScriptExectutionError.swift create mode 100644 SwiftSyntaxDevUtils/Sources/swift-syntax-dev-utils/common/SourceCodeGeneratorArguments.swift create mode 100644 SwiftSyntaxDevUtils/Sources/swift-syntax-dev-utils/common/SourceCodeGeneratorCommand.swift delete mode 100755 build-script.py diff --git a/SwiftSyntaxDevUtils/Package.swift b/SwiftSyntaxDevUtils/Package.swift new file mode 100644 index 00000000000..58bc77abd4f --- /dev/null +++ b/SwiftSyntaxDevUtils/Package.swift @@ -0,0 +1,33 @@ +// swift-tools-version:5.7 + +import PackageDescription +import Foundation + +let package = Package( + name: "swift-syntax-dev-utils", + platforms: [ + .macOS(.v10_15) + ], + products: [ + .executable(name: "swift-syntax-dev-utils", targets: ["swift-syntax-dev-utils"]) + ], + targets: [ + .executableTarget( + name: "swift-syntax-dev-utils", + dependencies: [ + .product(name: "ArgumentParser", package: "swift-argument-parser") + ] + ) + ] +) + +if ProcessInfo.processInfo.environment["SWIFTCI_USE_LOCAL_DEPS"] == nil { + // Building standalone. + package.dependencies += [ + .package(url: "https://github.com/apple/swift-argument-parser.git", from: "1.2.2") + ] +} else { + package.dependencies += [ + .package(path: "../../swift-argument-parser") + ] +} diff --git a/SwiftSyntaxDevUtils/README.md b/SwiftSyntaxDevUtils/README.md new file mode 100644 index 00000000000..b7a5f956826 --- /dev/null +++ b/SwiftSyntaxDevUtils/README.md @@ -0,0 +1,3 @@ +# swift-syntax-dev-utils + +Scripts to help build swift-syntax in CI. In most cases, you should not need to run these scripts yourself. The [Contributing Guide](../Contributing.md) contains information of how to build swift-syntax locally. diff --git a/SwiftSyntaxDevUtils/Sources/swift-syntax-dev-utils/SwiftSyntaxDevUtils.swift b/SwiftSyntaxDevUtils/Sources/swift-syntax-dev-utils/SwiftSyntaxDevUtils.swift new file mode 100644 index 00000000000..85209c9bfad --- /dev/null +++ b/SwiftSyntaxDevUtils/Sources/swift-syntax-dev-utils/SwiftSyntaxDevUtils.swift @@ -0,0 +1,34 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2014 - 2023 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// + +import ArgumentParser + +@main +struct SwiftSyntaxDevUtils: ParsableCommand { + + static var configuration: CommandConfiguration = CommandConfiguration( + abstract: """ + Build and test script for SwiftSyntax. + + The build script can also drive the test suite included in the SwiftSyntax + repo. This requires a custom build of the compiler project since it accesses + test utilities that are not shipped as part of the toolchains. See the Testing + section for arguments that need to be specified for this. + """, + subcommands: [ + Build.self, + GenerateSourceCode.self, + Test.self, + VerifySourceCode.self, + ] + ) +} diff --git a/SwiftSyntaxDevUtils/Sources/swift-syntax-dev-utils/commands/Build.swift b/SwiftSyntaxDevUtils/Sources/swift-syntax-dev-utils/commands/Build.swift new file mode 100644 index 00000000000..cc8000a1fc1 --- /dev/null +++ b/SwiftSyntaxDevUtils/Sources/swift-syntax-dev-utils/commands/Build.swift @@ -0,0 +1,24 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2014 - 2023 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// + +import ArgumentParser +import Foundation + +struct Build: ParsableCommand, BuildCommand { + @OptionGroup + var arguments: BuildArguments + + func run() throws { + try buildTarget(packageDir: Paths.packageDir, targetName: "SwiftSyntax-all") + try buildTarget(packageDir: Paths.examplesDir, targetName: "Examples-all") + } +} diff --git a/SwiftSyntaxDevUtils/Sources/swift-syntax-dev-utils/commands/GenerateSourceCode.swift b/SwiftSyntaxDevUtils/Sources/swift-syntax-dev-utils/commands/GenerateSourceCode.swift new file mode 100644 index 00000000000..6cfb48f9870 --- /dev/null +++ b/SwiftSyntaxDevUtils/Sources/swift-syntax-dev-utils/commands/GenerateSourceCode.swift @@ -0,0 +1,23 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2014 - 2023 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// + +import ArgumentParser +import Foundation + +struct GenerateSourceCode: ParsableCommand, SourceCodeGeneratorCommand { + @OptionGroup + var arguments: SourceCodeGeneratorArguments + + func run() throws { + try self.runCodeGeneration(sourceDir: Paths.sourcesDir) + } +} diff --git a/SwiftSyntaxDevUtils/Sources/swift-syntax-dev-utils/commands/Test.swift b/SwiftSyntaxDevUtils/Sources/swift-syntax-dev-utils/commands/Test.swift new file mode 100644 index 00000000000..2feae00728c --- /dev/null +++ b/SwiftSyntaxDevUtils/Sources/swift-syntax-dev-utils/commands/Test.swift @@ -0,0 +1,160 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2014 - 2023 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// + +import ArgumentParser +import Foundation + +struct Test: ParsableCommand, BuildCommand { + @OptionGroup + var arguments: BuildArguments + + @Flag(help: "Don't run lit-based tests") + var skipLitTests: Bool = false + + @Option( + help: """ + Path to the FileCheck executable that was built as part of the LLVM repository. + If not specified, it will be looked up from PATH. + """ + ) + var filecheckExec: String? + + func run() throws { + try buildExample(exampleName: "ExamplePlugin") + + try runTests() + + logSection("All tests passed") + } + + private func runTests() throws { + logSection("Running SwiftSyntax Tests") + + if !skipLitTests { + try runLitTests() + } + + try runXCTests() + } + + private func runLitTests() throws { + logSection("Running lit-based tests") + + guard FileManager.default.fileExists(atPath: Paths.litExec.path) else { + throw ScriptExectutionError( + message: """ + Error: Could not find lit.py. + Looking at '\(Paths.litExec.path)'. + + Make sure you have the llvm repo checked out next to the swift-syntax repo. + Refer to README.md for more information. + """ + ) + } + + let examplesBinPath = try findExamplesBinPath() + + var litCall = [ + Paths.litExec.path, + Paths.packageDir.appendingPathComponent("lit_tests").path, + ] + + if let filecheckExec { + litCall += ["--param", "FILECHECK=" + filecheckExec] + } + + litCall += ["--param", "EXAMPLES_BIN_PATH=" + examplesBinPath.path] + litCall += ["--param", "TOOLCHAIN=" + arguments.toolchain.path] + + // Print all failures + litCall += ["--verbose"] + // Don't show all commands if verbose is not enabled + if !arguments.verbose { + litCall += ["--succinct"] + } + + guard let pythonExec = Paths.python3Exec else { + throw ScriptExectutionError(message: "Didn't find python3 executable") + } + + let process = ProcessRunner( + executableURL: pythonExec, + arguments: litCall + ) + + let processOutput = try process.run(verbose: arguments.verbose) + + if !processOutput.stdout.isEmpty { + logSection("lit test stdout") + print(processOutput.stdout) + } + + if !processOutput.stderr.isEmpty { + logSection("lit test stderr") + print(processOutput.stderr) + } + } + + private func runXCTests() throws { + logSection("Running XCTests") + var swiftpmCallArguments: [String] = [] + + if arguments.verbose { + swiftpmCallArguments += ["--verbose"] + } + + swiftpmCallArguments += ["--test-product", "swift-syntaxPackageTests"] + + var additionalEnvironment: [String: String] = [:] + additionalEnvironment["SWIFT_BUILD_SCRIPT_ENVIRONMENT"] = "1" + + if arguments.enableRawSyntaxValidation { + additionalEnvironment["SWIFTSYNTAX_ENABLE_RAWSYNTAX_VALIDATION"] = "1" + } + + if arguments.enableTestFuzzing { + additionalEnvironment["SWIFTPARSER_ENABLE_ALTERNATE_TOKEN_INTROSPECTION"] = "1" + } + + // Tell other projects in the unified build to use local dependencies + additionalEnvironment["SWIFTCI_USE_LOCAL_DEPS"] = "1" + additionalEnvironment["SWIFT_SYNTAX_PARSER_LIB_SEARCH_PATH"] = + arguments.toolchain + .appendingPathComponent("lib") + .appendingPathComponent("swift") + .appendingPathComponent("macosx") + .path + + try invokeSwiftPM( + action: "test", + packageDir: Paths.packageDir, + additionalArguments: swiftpmCallArguments, + additionalEnvironment: additionalEnvironment + ) + } + + private func findSwiftpmBinPath(packageDir: URL) throws -> String { + return try invokeSwiftPM( + action: "build", + packageDir: packageDir, + additionalArguments: ["--show-bin-path"], + additionalEnvironment: [:] + ) + } + + /// This returns a path to the build examples folder. + /// Example: '/swift-syntax/Examples/.build/arm64-apple-macosx/debug + private func findExamplesBinPath() throws -> URL { + let stdOut = try findSwiftpmBinPath(packageDir: Paths.examplesDir) + return URL(fileURLWithPath: stdOut.trimmingCharacters(in: .whitespacesAndNewlines)) + } +} diff --git a/SwiftSyntaxDevUtils/Sources/swift-syntax-dev-utils/commands/VerifySourceCode.swift b/SwiftSyntaxDevUtils/Sources/swift-syntax-dev-utils/commands/VerifySourceCode.swift new file mode 100644 index 00000000000..0e9d2076d4f --- /dev/null +++ b/SwiftSyntaxDevUtils/Sources/swift-syntax-dev-utils/commands/VerifySourceCode.swift @@ -0,0 +1,67 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2014 - 2023 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// + +import ArgumentParser +import Foundation + +fileprivate var modules: [String] { + ["SwiftParser", "SwiftParserDiagnostics", "SwiftSyntax", "SwiftSyntaxBuilder"] +} + +struct VerifySourceCode: ParsableCommand, SourceCodeGeneratorCommand { + @OptionGroup + var arguments: SourceCodeGeneratorArguments + + func run() throws { + let tempDir = FileManager.default.temporaryDirectory + + try self.runCodeGeneration(sourceDir: tempDir) + + logSection("Verifing code generated files") + + guard let diffExec = Paths.diffExec else { + throw ScriptExectutionError(message: "Didn't find a diff execution path") + } + + for module in modules { + let selfGeneratedDir = tempDir.appendingPathComponent(module).appendingPathComponent("generated") + let userGeneratedDir = Paths.sourcesDir.appendingPathComponent(module).appendingPathComponent("generated") + + let process = ProcessRunner( + executableURL: diffExec, + arguments: [ + "--recursive", + "--exclude", + ".*", // Exclude dot files like .DS_Store + "--context=0", + selfGeneratedDir.path, + userGeneratedDir.path, + ] + ) + + let result = try process.run(verbose: arguments.verbose) + + if !result.stderr.isEmpty { + throw ScriptExectutionError( + message: """ + FAIL: code-generated files committed to repository do + not match generated ones. Please re-generate the + code-generated-files using the following command, open a PR to the + SwiftSyntax project and merge it alongside the main PR. + $ swift run swift-syntax-dev-utils generate-source-code + /path/to/toolchain.xctoolchain/usr + """ + ) + } + } + } +} diff --git a/SwiftSyntaxDevUtils/Sources/swift-syntax-dev-utils/common/BuildArguments.swift b/SwiftSyntaxDevUtils/Sources/swift-syntax-dev-utils/common/BuildArguments.swift new file mode 100644 index 00000000000..3198ce5166a --- /dev/null +++ b/SwiftSyntaxDevUtils/Sources/swift-syntax-dev-utils/common/BuildArguments.swift @@ -0,0 +1,70 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2014 - 2023 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// + +import ArgumentParser +import Foundation + +struct BuildArguments: ParsableArguments { + @Option( + help: "The path to the toolchain that shall be used to build SwiftSyntax.", + transform: URL.init(fileURLWithPath:) + ) + var toolchain: URL + + @Option( + help: """ + The directory in which build products shall be put. If omitted + a directory named ".build" will be put in the swift-syntax + directory. + """, + transform: URL.init(fileURLWithPath:) + ) + var buildDir: URL? + + @Option( + help: """ + Path to an Xcode workspace to create a unified build of SwiftSyntax with + other projects. + """, + transform: URL.init(fileURLWithPath:) + ) + var multirootDataFile: URL? + + @Flag(help: "Disable sandboxes when building with SwiftPM") + var disableSandbox: Bool = false + + @Flag(help: "Build in release mode.") + var release: Bool = false + + @Flag( + name: .customLong("enable-rawsyntax-validation"), + help: + """ + When constructing RawSyntax nodes validate that their layout matches that + defined in `CodeGeneration` and that TokenSyntax nodes have a `tokenKind` + matching the ones specified in `CodeGeneration`. + """ + ) + var enableRawSyntaxValidation: Bool = false + + @Flag( + help: """ + For each `assertParse` test, perform mutations of the test case based on + alternate token choices that the parser checks, validating that there are + no round-trip or assertion failures. + """ + ) + var enableTestFuzzing: Bool = false + + @Flag(help: "Enable verbose logging.") + var verbose: Bool = false +} diff --git a/SwiftSyntaxDevUtils/Sources/swift-syntax-dev-utils/common/BuildCommand.swift b/SwiftSyntaxDevUtils/Sources/swift-syntax-dev-utils/common/BuildCommand.swift new file mode 100644 index 00000000000..85328ef050a --- /dev/null +++ b/SwiftSyntaxDevUtils/Sources/swift-syntax-dev-utils/common/BuildCommand.swift @@ -0,0 +1,118 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2014 - 2023 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// + +import Foundation + +protocol BuildCommand { + var arguments: BuildArguments { get } +} + +extension BuildCommand { + func buildProduct(productName: String) throws { + logSection("Building product " + productName) + try build(packageDir: Paths.packageDir, name: productName, isProduct: true) + } + + func buildTarget(packageDir: URL, targetName: String) throws { + logSection("Building target " + targetName) + try build(packageDir: packageDir, name: targetName, isProduct: false) + } + + func buildExample(exampleName: String) throws { + logSection("Building example " + exampleName) + try build(packageDir: Paths.examplesDir, name: exampleName, isProduct: true) + } + + @discardableResult + func invokeSwiftPM( + action: String, + packageDir: URL, + additionalArguments: [String], + additionalEnvironment: [String: String] + ) throws -> String { + var args = [action] + args += ["--package-path", packageDir.path] + + if let buildDir = arguments.buildDir?.path { + args += ["--scratch-path", buildDir] + } + + #if !canImport(Darwin) + args += ["--enable-test-discovery"] + #endif + + if arguments.release { + args += ["--configuration", "release"] + } + + if let multirootDataFile = arguments.multirootDataFile?.path { + args += ["--multiroot-data-file", multirootDataFile] + } + + if arguments.disableSandbox { + args += ["--disable-sandbox"] + } + + if arguments.verbose { + args += ["--verbose"] + } + + args += additionalArguments + + let processRunner = ProcessRunner( + executableURL: arguments.toolchain.appendingPathComponent("bin").appendingPathComponent("swift"), + arguments: args, + additionalEnvironment: additionalEnvironment + ) + + let result = try processRunner.run(verbose: arguments.verbose) + + return result.stdout + } + + private func build(packageDir: URL, name: String, isProduct: Bool) throws { + let args: [String] + + if isProduct { + args = ["--product", name] + } else { + args = ["--target", name] + } + + var additionalEnvironment: [String: String] = [:] + additionalEnvironment["SWIFT_BUILD_SCRIPT_ENVIRONMENT"] = "1" + + if arguments.enableRawSyntaxValidation { + additionalEnvironment["SWIFTSYNTAX_ENABLE_RAWSYNTAX_VALIDATION"] = "1" + } + + if arguments.enableTestFuzzing { + additionalEnvironment["SWIFTPARSER_ENABLE_ALTERNATE_TOKEN_INTROSPECTION"] = "1" + } + + // Tell other projects in the unified build to use local dependencies + additionalEnvironment["SWIFTCI_USE_LOCAL_DEPS"] = "1" + additionalEnvironment["SWIFT_SYNTAX_PARSER_LIB_SEARCH_PATH"] = + arguments.toolchain + .appendingPathComponent("lib") + .appendingPathComponent("swift") + .appendingPathComponent("macos") + .path + + try invokeSwiftPM( + action: "build", + packageDir: packageDir, + additionalArguments: args, + additionalEnvironment: additionalEnvironment + ) + } +} diff --git a/SwiftSyntaxDevUtils/Sources/swift-syntax-dev-utils/common/Logger.swift b/SwiftSyntaxDevUtils/Sources/swift-syntax-dev-utils/common/Logger.swift new file mode 100644 index 00000000000..aaee888857e --- /dev/null +++ b/SwiftSyntaxDevUtils/Sources/swift-syntax-dev-utils/common/Logger.swift @@ -0,0 +1,31 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2014 - 2023 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// + +import Foundation + +func logSection(_ text: String) { + print("** \(text) **") +} + +func logProcessCommand(executableURL: URL?, arguments: [String]?) { + var message = "" + + if let executableURL = executableURL { + message += executableURL.absoluteString + } + + if let arguments = arguments { + message += " \(arguments.joined(separator: " "))" + } + + print(message) +} diff --git a/SwiftSyntaxDevUtils/Sources/swift-syntax-dev-utils/common/Paths.swift b/SwiftSyntaxDevUtils/Sources/swift-syntax-dev-utils/common/Paths.swift new file mode 100644 index 00000000000..a0b7a05a8ac --- /dev/null +++ b/SwiftSyntaxDevUtils/Sources/swift-syntax-dev-utils/common/Paths.swift @@ -0,0 +1,111 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2014 - 2023 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// + +import Foundation + +enum Paths { + static var packageDir: URL { + URL(fileURLWithPath: #file) + .deletingLastPathComponent() + .deletingLastPathComponent() + .deletingLastPathComponent() + .deletingLastPathComponent() + .deletingLastPathComponent() + } + + static var sourcesDir: URL { + packageDir + .appendingPathComponent("Sources") + } + + static var examplesDir: URL { + packageDir + .appendingPathComponent("Examples") + } + + static var codeGenerationDir: URL { + packageDir + .appendingPathComponent("CodeGeneration") + } + + static var workspaceDir: URL { + packageDir + .deletingLastPathComponent() + } + + static var llvmDir: URL { + workspaceDir + .appendingPathComponent("llvm-project") + .appendingPathComponent("llvm") + } + + static var litExec: URL { + llvmDir + .appendingPathComponent("utils") + .appendingPathComponent("lit") + .appendingPathComponent("lit.py") + } + + static var python3Exec: URL? { + return lookupExecutable(for: "python3") + } + + static var diffExec: URL? { + return lookupExecutable(for: "diff") + } + + private static var envSearchPaths: [URL] { + // Compute search paths from PATH variable. + #if os(Windows) + let pathSeparator: Character = ";" + #else + let pathSeparator: Character = ":" + #endif + return (paths ?? "") + .split(separator: pathSeparator) + .map(String.init) + .compactMap { pathString in + return URL(fileURLWithPath: pathString) + } + } + + private static var paths: String? { + #if os(Windows) + let pathArg = "Path" + #else + let pathArg = "PATH" + #endif + + return ProcessInfo.processInfo.environment[pathArg] + } + + private static func lookupExecutable(for filename: String) -> URL? { + return envSearchPaths.map { $0.appendingPathComponent(filename) } + .first(where: { $0.isExecutableFile }) + } +} + +fileprivate extension URL { + var isExecutableFile: Bool { + return (self.isFile(path) || self.isSymlink(path)) && FileManager.default.isExecutableFile(atPath: path) + } + + private func isFile(_ path: String) -> Bool { + let attrs = try? FileManager.default.attributesOfItem(atPath: path) + return attrs?[.type] as? FileAttributeType == .typeRegular + } + + private func isSymlink(_ path: String) -> Bool { + let attrs = try? FileManager.default.attributesOfItem(atPath: path) + return attrs?[.type] as? FileAttributeType == .typeSymbolicLink + } +} diff --git a/SwiftSyntaxDevUtils/Sources/swift-syntax-dev-utils/common/ProcessRunner.swift b/SwiftSyntaxDevUtils/Sources/swift-syntax-dev-utils/common/ProcessRunner.swift new file mode 100644 index 00000000000..329fd4afec1 --- /dev/null +++ b/SwiftSyntaxDevUtils/Sources/swift-syntax-dev-utils/common/ProcessRunner.swift @@ -0,0 +1,122 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2014 - 2018 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// + +import Foundation + +/// Provides convenience APIs for launching and gathering output from a subprocess +public class ProcessRunner { + private static let serialQueue = DispatchQueue(label: "\(ProcessRunner.self)") + + private let process: Process + + public init( + executableURL: URL, + arguments: [String], + additionalEnvironment: [String: String] = [:] + ) { + process = Process() + process.executableURL = executableURL + process.arguments = arguments + process.environment = additionalEnvironment.merging(ProcessInfo.processInfo.environment) { (additional, _) in additional } + } + + @discardableResult + public func run( + verbose: Bool, + captureStdout: Bool = true, + captureStderr: Bool = true + ) throws -> ProcessResult { + if verbose { + logProcessCommand(executableURL: process.executableURL, arguments: process.arguments) + } + + let group = DispatchGroup() + + var stdoutData = Data() + if captureStdout { + let outPipe = Pipe() + process.standardOutput = outPipe + addHandler(pipe: outPipe, group: group) { stdoutData.append($0) } + } + + var stderrData = Data() + if captureStderr { + let errPipe = Pipe() + process.standardError = errPipe + addHandler(pipe: errPipe, group: group) { stderrData.append($0) } + } + + try process.run() + process.waitUntilExit() + if captureStdout || captureStderr { + // Make sure we've received all stdout/stderr + group.wait() + } + + guard let stdoutString = String(data: stdoutData, encoding: .utf8) else { + throw FailedToDecodeUTF8Error(data: stdoutData) + } + guard let stderrString = String(data: stderrData, encoding: .utf8) else { + throw FailedToDecodeUTF8Error(data: stderrData) + } + + guard process.terminationStatus == 0 else { + throw NonZeroExitCodeError( + stdout: stdoutString, + stderr: stderrString, + exitCode: Int(process.terminationStatus) + ) + } + + return ProcessResult( + stdout: stdoutString, + stderr: stderrString + ) + } + + private func addHandler( + pipe: Pipe, + group: DispatchGroup, + addData: @escaping (Data) -> Void + ) { + group.enter() + pipe.fileHandleForReading.readabilityHandler = { fileHandle in + // Apparently using availableData can cause various issues + let newData = fileHandle.readData(ofLength: Int.max) + if newData.count == 0 { + pipe.fileHandleForReading.readabilityHandler = nil; + group.leave() + } else { + addData(newData) + } + } + } +} + +/// The exit code and output (if redirected) from a subprocess that has +/// terminated +public struct ProcessResult { + public let stdout: String + public let stderr: String +} + +/// Error thrown if a process terminates with a non-zero exit code. +struct NonZeroExitCodeError: Error { + let stdout: String + let stderr: String + let exitCode: Int +} + +/// Error thrown if `stdout` or `stderr` could not be decoded as UTF-8. +struct FailedToDecodeUTF8Error: Error { + let data: Data +} diff --git a/SwiftSyntaxDevUtils/Sources/swift-syntax-dev-utils/common/ScriptExectutionError.swift b/SwiftSyntaxDevUtils/Sources/swift-syntax-dev-utils/common/ScriptExectutionError.swift new file mode 100644 index 00000000000..0a5f6c013b9 --- /dev/null +++ b/SwiftSyntaxDevUtils/Sources/swift-syntax-dev-utils/common/ScriptExectutionError.swift @@ -0,0 +1,21 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2014 - 2023 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// + +struct ScriptExectutionError: Error { + let message: String +} + +extension ScriptExectutionError: CustomStringConvertible { + var description: String { + message + } +} diff --git a/SwiftSyntaxDevUtils/Sources/swift-syntax-dev-utils/common/SourceCodeGeneratorArguments.swift b/SwiftSyntaxDevUtils/Sources/swift-syntax-dev-utils/common/SourceCodeGeneratorArguments.swift new file mode 100644 index 00000000000..cdfd7ad80e2 --- /dev/null +++ b/SwiftSyntaxDevUtils/Sources/swift-syntax-dev-utils/common/SourceCodeGeneratorArguments.swift @@ -0,0 +1,25 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2014 - 2023 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// + +import ArgumentParser +import Foundation + +struct SourceCodeGeneratorArguments: ParsableArguments { + @Option( + help: "The path to the toolchain that shall be used to build SwiftSyntax.", + transform: URL.init(fileURLWithPath:) + ) + var toolchain: URL + + @Flag(help: "Enable verbose logging.") + var verbose: Bool = false +} diff --git a/SwiftSyntaxDevUtils/Sources/swift-syntax-dev-utils/common/SourceCodeGeneratorCommand.swift b/SwiftSyntaxDevUtils/Sources/swift-syntax-dev-utils/common/SourceCodeGeneratorCommand.swift new file mode 100644 index 00000000000..44bacb934d7 --- /dev/null +++ b/SwiftSyntaxDevUtils/Sources/swift-syntax-dev-utils/common/SourceCodeGeneratorCommand.swift @@ -0,0 +1,49 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2014 - 2023 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// + +import Foundation +import ArgumentParser +import Dispatch +import Foundation + +protocol SourceCodeGeneratorCommand { + var arguments: SourceCodeGeneratorArguments { get } +} + +extension SourceCodeGeneratorCommand { + func runCodeGeneration(sourceDir: URL) throws { + logSection("Running code generation") + + var args = [ + "run", + "--package-path", Paths.codeGenerationDir.relativePath, + "generate-swiftsyntax", sourceDir.relativePath, + ] + + if arguments.verbose { + args += ["--verbose"] + } + + let additionalEnvironment = [ + "SWIFT_BUILD_SCRIPT_ENVIRONMENT": "1", + "SWIFTSYNTAX_ENABLE_RAWSYNTAX_VALIDATION": "1", + ] + + let process = ProcessRunner( + executableURL: arguments.toolchain.appendingPathComponent("bin").appendingPathComponent("swift"), + arguments: args, + additionalEnvironment: additionalEnvironment + ) + + try process.run(verbose: arguments.verbose) + } +} diff --git a/build-script.py b/build-script.py deleted file mode 100755 index 980e0fbc4e9..00000000000 --- a/build-script.py +++ /dev/null @@ -1,721 +0,0 @@ -#!/usr/bin/env python3 - -import argparse -import os -import platform -import subprocess -import sys -import tempfile -from typing import Dict, List, Optional - - -# ----------------------------------------------------------------------------- -# Constants - -PACKAGE_DIR = os.path.dirname(os.path.realpath(__file__)) -WORKSPACE_DIR = os.path.dirname(PACKAGE_DIR) -EXAMPLES_DIR = os.path.join(PACKAGE_DIR, "Examples") -SOURCES_DIR = os.path.join(PACKAGE_DIR, "Sources") -TESTS_DIR = os.path.join(PACKAGE_DIR, "Tests") -SWIFTIDEUTILS_DIR = os.path.join(SOURCES_DIR, "SwiftIDEUtils") -SWIFTSYNTAX_DIR = os.path.join(SOURCES_DIR, "SwiftSyntax") -SWIFTSYNTAX_DOCUMENTATION_DIR = \ - os.path.join(SWIFTSYNTAX_DIR, "Documentation.docc") -SWIFTBASICFORMAT_DIR = os.path.join(SOURCES_DIR, "SwiftBasicFormat") -SWIFTSYNTAXBUILDER_DIR = os.path.join(SOURCES_DIR, "SwiftSyntaxBuilder") -SWIFTPARSER_DIR = os.path.join(SOURCES_DIR, "SwiftParser") - -CODE_GENERATION_DIR = os.path.join(PACKAGE_DIR, "CodeGeneration") -SYNTAXSUPPORT_DIR = \ - os.path.join(CODE_GENERATION_DIR, "Sources", "SyntaxSupport") - -LLVM_DIR = os.path.join(WORKSPACE_DIR, "llvm-project", "llvm") -SWIFT_DIR = os.path.join(WORKSPACE_DIR, "swift") - -LIT_EXEC = os.path.join(LLVM_DIR, "utils", "lit", "lit.py") - -GROUP_INFO_PATH = os.path.join(PACKAGE_DIR, "utils", "group.json") - - -def fail_for_called_process_error( - succinct_description: str, - error: subprocess.CalledProcessError -) -> None: - printerr(f"FAIL: {succinct_description}") - printerr(f"Executing: {escapeCmd(error.cmd)}") - printerr(error.output) - raise SystemExit(1) - - -# ----------------------------------------------------------------------------- -# Xcode Projects Generation - - -def xcode_gen(config: str) -> None: - print("** Generate SwiftSyntax as an Xcode project **") - os.chdir(PACKAGE_DIR) - swiftpm_call = ["swift", "package", "generate-xcodeproj"] - if config: - swiftpm_call.extend(["--xcconfig-overrides", config]) - check_call(swiftpm_call) - - -# ----------------------------------------------------------------------------- -# Helpers - - -def printerr(message: str) -> None: - print(message, file=sys.stderr) - - -def note(message: str) -> None: - print("--- %s: note: %s" % (os.path.basename(sys.argv[0]), message)) - sys.stdout.flush() - - -def fatal_error(message: str) -> None: - printerr(message) - raise SystemExit(1) - - -def escapeCmdArg(arg: str) -> str: - if '"' in arg or " " in arg: - return '"%s"' % arg.replace('"', '\\"') - else: - return arg - - -def escapeCmd(cmd: List[str]) -> str: - return " ".join([escapeCmdArg(arg) for arg in cmd]) - - -def call(cmd: List[str], env: Dict[str, str] = dict(os.environ), - stdout: Optional[int] = None, stderr: Optional[int] = subprocess.STDOUT, - verbose: bool = False) -> int: - if verbose: - print(escapeCmd(cmd)) - process = subprocess.Popen(cmd, env=env, stdout=stdout, stderr=stderr) - process.wait() - - return process.returncode - - -def check_call(cmd: List[str], cwd: Optional[str] = None, - env: Dict[str, str] = dict(os.environ), verbose: bool = False) -> None: - if verbose: - print(escapeCmd(cmd)) - subprocess.check_call(cmd, cwd=cwd, env=env, stderr=subprocess.STDOUT) - - -def realpath(path: Optional[str]) -> Optional[str]: - if path is None: - return None - return os.path.realpath(path) - - -# ----------------------------------------------------------------------------- -# Generating Files - - -def run_code_generation( - source_dir: str, - toolchain: str, - verbose: bool -) -> None: - print("** Running code generation **") - - swift_exec = os.path.join(toolchain, "bin", "swift") - - swiftpm_call = [ - swift_exec, 'run', - "--package-path", CODE_GENERATION_DIR, - "generate-swiftsyntax", source_dir - ] - - if verbose: - swiftpm_call.extend(["--verbose"]) - - env = dict(os.environ) - env["SWIFT_BUILD_SCRIPT_ENVIRONMENT"] = "1" - env["SWIFTSYNTAX_ENABLE_RAWSYNTAX_VALIDATION"] = "1" - env["SWIFTCI_USE_LOCAL_DEPS"] = "1" - check_call(swiftpm_call, env=env, verbose=verbose) - - -# ----------------------------------------------------------------------------- -# Building SwiftSyntax - - -def get_swiftpm_invocation( - toolchain: str, action: str, package_dir: str, build_dir: Optional[str], - multiroot_data_file: Optional[str], release: bool -) -> List[str]: - swift_exec = os.path.join(toolchain, "bin", "swift") - - swiftpm_call = [swift_exec, action] - swiftpm_call.extend(["--package-path", package_dir]) - if platform.system() != "Darwin": - swiftpm_call.extend(["--enable-test-discovery"]) - if release: - swiftpm_call.extend(["--configuration", "release"]) - if build_dir: - swiftpm_call.extend(["--scratch-path", build_dir]) - if multiroot_data_file: - swiftpm_call.extend(["--multiroot-data-file", multiroot_data_file]) - - return swiftpm_call - - -class Builder(object): - verbose: bool - toolchain: str - build_dir: Optional[str] - multiroot_data_file: Optional[str] - release: bool - enable_rawsyntax_validation: bool - enable_test_fuzzing: bool - disable_sandbox: bool - - def __init__( - self, - toolchain: str, - build_dir: Optional[str], - multiroot_data_file: Optional[str], - release: bool, - enable_rawsyntax_validation: bool, - enable_test_fuzzing: bool, - verbose: bool, - disable_sandbox: bool = False, - ) -> None: - self.build_dir = build_dir - self.multiroot_data_file = multiroot_data_file - self.release = release - self.enable_rawsyntax_validation = enable_rawsyntax_validation - self.enable_test_fuzzing = enable_test_fuzzing - self.disable_sandbox = disable_sandbox - self.verbose = verbose - self.toolchain = toolchain - - def __get_swiftpm_invocation(self, package_dir: str) -> List[str]: - invocation = get_swiftpm_invocation( - self.toolchain, - "build", - package_dir, - self.build_dir, - self.multiroot_data_file, - self.release - ) - if self.disable_sandbox: - invocation.append("--disable-sandbox") - if self.verbose: - invocation.append("--verbose") - return invocation - - def buildProduct(self, product_name: str) -> None: - print("** Building product " + product_name + " **") - self.__build(PACKAGE_DIR, product_name, is_product=True) - - def buildTarget(self, package_dir: str, target_name: str) -> None: - print("** Building target " + target_name + " **") - self.__build(package_dir, target_name, is_product=False) - - def buildExample(self, example_name: str) -> None: - print("** Building example " + example_name + " **") - self.__build(EXAMPLES_DIR, example_name, is_product=True) - - def __build(self, package_dir: str, name: str, is_product: bool) -> None: - command = list(self.__get_swiftpm_invocation(package_dir)) - command.extend(["--product" if is_product else "--target", name]) - - env = dict(os.environ) - env["SWIFT_BUILD_SCRIPT_ENVIRONMENT"] = "1" - if self.enable_rawsyntax_validation: - env["SWIFTSYNTAX_ENABLE_RAWSYNTAX_VALIDATION"] = "1" - if self.enable_test_fuzzing: - env["SWIFTPARSER_ENABLE_ALTERNATE_TOKEN_INTROSPECTION"] = "1" - # Tell other projects in the unified build to use local dependencies - env["SWIFTCI_USE_LOCAL_DEPS"] = "1" - env["SWIFT_SYNTAX_PARSER_LIB_SEARCH_PATH"] = \ - os.path.join(self.toolchain, "lib", "swift", "macosx") - - check_call(command, env=env, verbose=self.verbose) - - -# ----------------------------------------------------------------------------- -# Testing - - -def verify_code_generated_files( - toolchain: str, verbose: bool -) -> None: - self_temp_dir = tempfile.mkdtemp() - - try: - run_code_generation( - source_dir=self_temp_dir, - toolchain=toolchain, - verbose=verbose - ) - except subprocess.CalledProcessError as e: - fail_for_called_process_error( - "Source generation using SwiftSyntaxBuilder failed", - e - ) - - print("** Verifing code generated files **") - - for module in ["SwiftParser", "SwiftParserDiagnostics", \ - "SwiftSyntax", "SwiftSyntaxBuilder"]: - self_generated_dir = os.path.join(self_temp_dir, module, "generated") - user_generated_dir = os.path.join(SOURCES_DIR, module, "generated") - check_generated_files_match(self_generated_dir, user_generated_dir) - - -def check_generated_files_match(self_generated_dir: str, - user_generated_dir: str) -> None: - command = [ - "diff", - "--recursive", - "--exclude", - ".*", # Exclude dot files like .DS_Store - "--context=0", - self_generated_dir, - user_generated_dir, - ] - check_call(command) - - -def run_code_generation_tests( - toolchain: str, - verbose: bool -) -> None: - print("** Running CodeGeneration tests **") - - swift_exec = os.path.join(toolchain, "bin", "swift") - - swiftpm_call = [ - swift_exec, 'test', - "--package-path", CODE_GENERATION_DIR - ] - - if verbose: - swiftpm_call.extend(["--verbose"]) - - env = dict(os.environ) - env["SWIFT_BUILD_SCRIPT_ENVIRONMENT"] = "1" - env["SWIFTSYNTAX_ENABLE_RAWSYNTAX_VALIDATION"] = "1" - env["SWIFTCI_USE_LOCAL_DEPS"] = "1" - check_call(swiftpm_call, env=env, verbose=verbose) - - -def run_tests( - toolchain: str, - build_dir: Optional[str], - multiroot_data_file: Optional[str], - release: bool, - enable_rawsyntax_validation: bool, - enable_test_fuzzing: bool, - filecheck_exec: Optional[str], - skip_lit_tests: bool, - verbose: bool -) -> None: - print("** Running SwiftSyntax Tests **") - - if not skip_lit_tests: - run_lit_tests( - toolchain=toolchain, - build_dir=build_dir, - release=release, - filecheck_exec=filecheck_exec, - verbose=verbose, - ) - - run_xctests( - package_dir=PACKAGE_DIR, - test_product="swift-syntaxPackageTests", - toolchain=toolchain, - build_dir=build_dir, - multiroot_data_file=multiroot_data_file, - release=release, - enable_rawsyntax_validation=enable_rawsyntax_validation, - enable_test_fuzzing=enable_test_fuzzing, - verbose=verbose, - ) - - run_code_generation_tests( - toolchain=toolchain, - verbose=verbose, - ) - -# ----------------------------------------------------------------------------- -# Lit Tests - - -def check_lit_exec() -> None: - if not os.path.exists(LIT_EXEC): - fatal_error( - """ -Error: Could not find lit.py. -Looking at '%s'. - -Make sure you have the llvm repo checked out next to the swift-syntax repo. -Refer to README.md for more information. -""" - % LIT_EXEC - ) - - -def find_swiftpm_bin_path( - package_dir: str, toolchain: str, build_dir: Optional[str], release: bool -) -> str: - swiftpm_call = get_swiftpm_invocation( - toolchain=toolchain, - action="build", - package_dir=package_dir, - build_dir=build_dir, - multiroot_data_file=None, - release=release, - ) - swiftpm_call.extend(["--show-bin-path"]) - - bin_dir = subprocess.check_output(swiftpm_call) - return bin_dir.strip().decode('utf-8') - - -def find_product_bin_path( - toolchain: str, build_dir: Optional[str], release: bool -) -> str: - return find_swiftpm_bin_path(PACKAGE_DIR, toolchain, build_dir, release) - - -def find_examples_bin_path( - toolchain: str, build_dir: Optional[str], release: bool -) -> str: - return find_swiftpm_bin_path(EXAMPLES_DIR, toolchain, build_dir, release) - - -def run_lit_tests(toolchain: str, build_dir: Optional[str], release: bool, - filecheck_exec: Optional[str], verbose: bool) -> None: - print("** Running lit-based tests **") - - check_lit_exec() - - product_bin_path = find_product_bin_path( - toolchain=toolchain, build_dir=build_dir, release=release) - examples_bin_path = find_examples_bin_path( - toolchain=toolchain, build_dir=build_dir, release=release) - - lit_call = ["python3", LIT_EXEC] - lit_call.append(os.path.join(PACKAGE_DIR, "lit_tests")) - - if filecheck_exec: - lit_call.extend(["--param", "FILECHECK=" + filecheck_exec]) - lit_call.extend(["--param", "EXAMPLES_BIN_PATH=" + examples_bin_path]) - lit_call.extend(["--param", "TOOLCHAIN=" + toolchain]) - - # Print all failures - lit_call.extend(["--verbose"]) - # Don't show all commands if verbose is not enabled - if not verbose: - lit_call.extend(["--succinct"]) - check_call(lit_call, verbose=verbose) - - -# ----------------------------------------------------------------------------- -# XCTest Tests - - -def run_xctests( - package_dir: str, - test_product: str, - toolchain: str, - build_dir: Optional[str], - multiroot_data_file: Optional[str], - release: bool, - enable_rawsyntax_validation: bool, - enable_test_fuzzing: bool, - verbose: bool -) -> None: - print("** Running XCTests **") - swiftpm_call = get_swiftpm_invocation( - toolchain=toolchain, - action="test", - package_dir=package_dir, - build_dir=build_dir, - multiroot_data_file=multiroot_data_file, - release=release, - ) - - if verbose: - swiftpm_call.extend(["--verbose"]) - - swiftpm_call.extend(["--test-product", test_product]) - - env = dict(os.environ) - env["SWIFT_BUILD_SCRIPT_ENVIRONMENT"] = "1" - if enable_rawsyntax_validation: - env["SWIFTSYNTAX_ENABLE_RAWSYNTAX_VALIDATION"] = "1" - if enable_test_fuzzing: - env["SWIFTPARSER_ENABLE_ALTERNATE_TOKEN_INTROSPECTION"] = "1" - # Tell other projects in the unified build to use local dependencies - env["SWIFTCI_USE_LOCAL_DEPS"] = "1" - env["SWIFT_SYNTAX_PARSER_LIB_SEARCH_PATH"] = \ - os.path.join(toolchain, "lib", "swift", "macosx") - - check_call(swiftpm_call, env=env, verbose=verbose) - -# ----------------------------------------------------------------------------- -# Arugment Parsing functions - - -def generate_xcodeproj(args: argparse.Namespace) -> None: - xcode_gen(config=args.xcconfig_path) - - -def generate_source_code_command(args: argparse.Namespace) -> None: - try: - run_code_generation( - source_dir=SOURCES_DIR, - toolchain=args.toolchain, - verbose=args.verbose - ) - except subprocess.CalledProcessError as e: - fail_for_called_process_error( - "Source generation using SwiftSyntaxBuilder failed", e) - - -def verify_source_code_command(args: argparse.Namespace) -> None: - try: - verify_code_generated_files( - toolchain=args.toolchain, - verbose=args.verbose, - ) - except subprocess.CalledProcessError: - printerr( - "FAIL: code-generated files committed to repository do " + - "not match generated ones. Please re-generate the " + - "code-generated-files using the following command, open a PR to the " + - "SwiftSyntax project and merge it alongside the main PR." + - "$ swift-syntax/build-script.py generate-source-code " + - "--toolchain /path/to/toolchain.xctoolchain/usr" - ) - raise SystemExit(1) - - -def build_command(args: argparse.Namespace) -> None: - try: - builder = Builder( - toolchain=realpath(args.toolchain), # pyright: ignore - build_dir=realpath(args.build_dir), - multiroot_data_file=args.multiroot_data_file, - release=args.release, - enable_rawsyntax_validation=args.enable_rawsyntax_validation, - enable_test_fuzzing=args.enable_test_fuzzing, - verbose=args.verbose, - disable_sandbox=args.disable_sandbox, - ) - builder.buildTarget(PACKAGE_DIR, "SwiftSyntax-all") - builder.buildTarget(EXAMPLES_DIR, "Examples-all") - except subprocess.CalledProcessError as e: - fail_for_called_process_error("Building SwiftSyntax failed", e) - - -def test_command(args: argparse.Namespace) -> None: - try: - builder = Builder( - toolchain=realpath(args.toolchain), # pyright: ignore - build_dir=realpath(args.build_dir), - multiroot_data_file=args.multiroot_data_file, - release=args.release, - enable_rawsyntax_validation=args.enable_rawsyntax_validation, - enable_test_fuzzing=args.enable_test_fuzzing, - verbose=args.verbose, - disable_sandbox=args.disable_sandbox, - ) - - builder.buildExample("ExamplePlugin") - - run_tests( - toolchain=realpath(args.toolchain), # pyright: ignore - build_dir=realpath(args.build_dir), - multiroot_data_file=args.multiroot_data_file, - release=args.release, - enable_rawsyntax_validation=args.enable_rawsyntax_validation, - enable_test_fuzzing=args.enable_test_fuzzing, - filecheck_exec=realpath(args.filecheck_exec), - skip_lit_tests=args.skip_lit_tests, - verbose=args.verbose, - ) - print("** All tests passed **") - except subprocess.CalledProcessError as e: - fail_for_called_process_error("Running tests failed", e) - - -# ----------------------------------------------------------------------------- -# Argument Parsing - -_DESCRIPTION = """ -Build and test script for SwiftSyntax. - -The build script can also drive the test suite included in the SwiftSyntax -repo. This requires a custom build of the compiler project since it accesses -test utilities that are not shipped as part of the toolchains. See the Testing -section for arguments that need to be specified for this. -""" - - -def parse_args() -> argparse.Namespace: - def add_default_build_arguments(parser: argparse.ArgumentParser) -> None: - parser.add_argument( - "-r", "--release", action="store_true", help="Build in release mode." - ) - - parser.add_argument( - "--build-dir", - default=None, - help=""" - The directory in which build products shall be put. If omitted - a directory named ".build" will be put in the swift-syntax - directory. - """, - ) - - parser.add_argument( - "--disable-sandbox", - action="store_true", - help="Disable sandboxes when building with SwiftPM", - ) - - parser.add_argument( - "--multiroot-data-file", - help=""" - Path to an Xcode workspace to create a unified build of SwiftSyntax with - other projects. - """, - ) - - parser.add_argument( - "--enable-rawsyntax-validation", - action="store_true", - help=""" - When constructing RawSyntax nodes validate that their layout matches that - defined in `CodeGeneration` and that TokenSyntax nodes have a `tokenKind` - matching the ones specified in `CodeGeneration`. - """ - ) - - parser.add_argument( - "--enable-test-fuzzing", - action="store_true", - help=""" - For each `assertParse` test, perform mutations of the test case based on - alternate token choices that the parser checks, validating that there are - no round-trip or assertion failures. - """ - ) - - parser.add_argument( - "--toolchain", - required=True, - help="The path to the toolchain that shall be used to build SwiftSyntax.", - ) - - parser.add_argument( - "-v", "--verbose", action="store_true", help="Enable verbose logging." - ) - - def add_generate_source_code_args(parser: argparse.ArgumentParser) -> None: - parser.add_argument( - "--toolchain", - required=True, - help="The path to the toolchain that shall be used to build SwiftSyntax.", - ) - - parser.add_argument( - "-v", "--verbose", action="store_true", help="Enable verbose logging." - ) - - parser = argparse.ArgumentParser( - formatter_class=argparse.RawDescriptionHelpFormatter, description=_DESCRIPTION - ) - - # ------------------------------------------------------------------------- - # Shared arguments - - sub_parsers = parser.add_subparsers() - - # ------------------------------------------------------------------------- - generate_xcodeproj_parser = sub_parsers.add_parser( - "generate-xcodeproj", - help="Generate an Xcode project for SwiftSyntax." - ) - generate_xcodeproj_parser.set_defaults(func=generate_xcodeproj) - - generate_xcodeproj_parser.add_argument( - "--xcconfig-path", - help="The path to an xcconfig file for generating Xcode projct.", - ) - - generate_xcodeproj_parser.add_argument( - "-v", "--verbose", action="store_true", help="Enable verbose logging." - ) - - # ------------------------------------------------------------------------- - generate_source_code_parser = sub_parsers.add_parser("generate-source-code") - generate_source_code_parser.set_defaults(func=generate_source_code_command) - - add_generate_source_code_args(generate_source_code_parser) - - generate_source_code_parser.add_argument( - "--add-source-locations", - action="store_true", - help=""" - Insert ###sourceLocation comments in generated code for line-directive. - """, - ) - - # ------------------------------------------------------------------------- - build_parser = sub_parsers.add_parser("build") - build_parser.set_defaults(func=build_command) - - add_default_build_arguments(build_parser) - - # ------------------------------------------------------------------------- - test_parser = sub_parsers.add_parser("test") - test_parser.set_defaults(func=test_command) - - add_default_build_arguments(test_parser) - - test_parser.add_argument( - "--skip-lit-tests", action="store_true", - help="Don't run lit-based tests" - ) - - test_parser.add_argument( - "--filecheck-exec", - default=None, - help=""" - Path to the FileCheck executable that was built as part of the LLVM repository. - If not specified, it will be looked up from PATH. - """, - ) - - # ------------------------------------------------------------------------- - verify_source_code_parser = sub_parsers.add_parser("verify-source-code") - verify_source_code_parser.set_defaults(func=verify_source_code_command) - - add_generate_source_code_args(verify_source_code_parser) - - return parser.parse_args() - - -# ----------------------------------------------------------------------------- - - -def main() -> None: - args = parse_args() - args.func(args) - - -if __name__ == "__main__": - main()