Skip to content

Commit d7b3c42

Browse files
kimdvkimdv
committed
Migrate build script to Swift
1 parent 3ed9dcf commit d7b3c42

15 files changed

+858
-0
lines changed

‎SwiftSyntaxDevUtils/Package.swift

Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,33 @@
1+
// swift-tools-version:5.7
2+
3+
import PackageDescription
4+
import Foundation
5+
6+
let package = Package(
7+
name: "SwiftSyntaxDevUtils",
8+
platforms: [
9+
.macOS(.v10_15)
10+
],
11+
products: [
12+
.executable(name: "swift-syntax-dev-utils", targets: ["swift-syntax-dev-utils"])
13+
],
14+
targets: [
15+
.executableTarget(
16+
name: "swift-syntax-dev-utils",
17+
dependencies: [
18+
.product(name: "ArgumentParser", package: "swift-argument-parser")
19+
]
20+
)
21+
]
22+
)
23+
24+
if ProcessInfo.processInfo.environment["SWIFTCI_USE_LOCAL_DEPS"] == nil {
25+
// Building standalone.
26+
package.dependencies += [
27+
.package(url: "https://github.com/apple/swift-argument-parser.git", from: "1.2.2")
28+
]
29+
} else {
30+
package.dependencies += [
31+
.package(path: "../../swift-argument-parser")
32+
]
33+
}

‎SwiftSyntaxDevUtils/README.md

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,21 @@
1+
# SwiftSyntaxCLI
2+
3+
## Code-Generation for SwiftSyntax
4+
5+
This directory contains file to generate source code that is part of the SwiftSyntax package. If you are looking to generate Swift code yourself, you might be interested in [SwiftSyntaxBuilder](../Sources/SwiftSyntaxBuilder).
6+
7+
Some source code inside SwiftSyntax is generated using [SwiftSyntaxBuilder](../Sources/SwiftSyntaxBuilder), a Swift library whose purpose is to generate Swift code using Swift itself. This kind of code generation is performed by the Swift package defined in this directory.
8+
9+
This directory is a standalone package that uses HEAD of the current branch. This guarantees that when `generate-source-code` is run, it can't break its own build when run multiple times without committing.
10+
This means that `SwiftSyntaxCLI` will build against your latest local commit of SwiftSyntax. If you are making changes to `SwiftSyntax` that affect how code is being generated, commit your SwiftSyntax changes (pushing is not necessary) and re-generate files afterwards.
11+
12+
To re-generate the files after changing `SwiftSyntaxCLI` run the `swift-syntax-cli`
13+
target of `SwiftSyntaxCLI` and pass `path/to/swift-syntax/Sources` as the argument.
14+
15+
On the command line, this would be
16+
```bash
17+
swift run --package-path SwiftSyntaxCLI swift-syntax-cli generate-source-code Sources
18+
```
19+
20+
Or if you open the `SwiftSyntaxCLI` package in Xcode, you can add the
21+
`swift-syntax-cli generate-source-code ../Sources` arguments using Product -> Scheme -> Edit Scheme…

‎SwiftSyntaxDevUtils/Sources/swift-syntax-dev-utils/SwiftSyntaxCLI.swift

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,34 @@
1+
//===----------------------------------------------------------------------===//
2+
//
3+
// This source file is part of the Swift.org open source project
4+
//
5+
// Copyright (c) 2014 - 2023 Apple Inc. and the Swift project authors
6+
// Licensed under Apache License v2.0 with Runtime Library Exception
7+
//
8+
// See https://swift.org/LICENSE.txt for license information
9+
// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
10+
//
11+
//===----------------------------------------------------------------------===//
12+
13+
import ArgumentParser
14+
15+
@main
16+
struct SwiftSyntaxCLI: ParsableCommand {
17+
18+
static var configuration: CommandConfiguration = CommandConfiguration(
19+
abstract: """
20+
Build and test script for SwiftSyntax.
21+
22+
The build script can also drive the test suite included in the SwiftSyntax
23+
repo. This requires a custom build of the compiler project since it accesses
24+
test utilities that are not shipped as part of the toolchains. See the Testing
25+
section for arguments that need to be specified for this.
26+
""",
27+
subcommands: [
28+
Build.self,
29+
GenerateSourceCode.self,
30+
Test.self,
31+
VerifySourceCode.self,
32+
]
33+
)
34+
}

‎SwiftSyntaxDevUtils/Sources/swift-syntax-dev-utils/commands/Build.swift

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,27 @@
1+
//===----------------------------------------------------------------------===//
2+
//
3+
// This source file is part of the Swift.org open source project
4+
//
5+
// Copyright (c) 2014 - 2023 Apple Inc. and the Swift project authors
6+
// Licensed under Apache License v2.0 with Runtime Library Exception
7+
//
8+
// See https://swift.org/LICENSE.txt for license information
9+
// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
10+
//
11+
//===----------------------------------------------------------------------===//
12+
13+
import ArgumentParser
14+
import Foundation
15+
16+
struct Build: ParsableCommand, Builder {
17+
@OptionGroup
18+
var arguments: BuilderArguments
19+
20+
func run() throws {
21+
try buildProduct(productName: "SwiftSyntax-all")
22+
23+
// Build examples
24+
try buildExample(exampleName: "AddOneToIntegerLiterals")
25+
try buildExample(exampleName: "CodeGenerationUsingSwiftSyntaxBuilder")
26+
}
27+
}

‎SwiftSyntaxDevUtils/Sources/swift-syntax-dev-utils/commands/GenerateSourceCode.swift

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
//===----------------------------------------------------------------------===//
2+
//
3+
// This source file is part of the Swift.org open source project
4+
//
5+
// Copyright (c) 2014 - 2023 Apple Inc. and the Swift project authors
6+
// Licensed under Apache License v2.0 with Runtime Library Exception
7+
//
8+
// See https://swift.org/LICENSE.txt for license information
9+
// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
10+
//
11+
//===----------------------------------------------------------------------===//
12+
13+
import ArgumentParser
14+
import Foundation
15+
16+
struct GenerateSourceCode: ParsableCommand, SourceCodeGeneratorCommand {
17+
@OptionGroup
18+
var arguments: SourceCodeGeneratorArguments
19+
20+
func run() throws {
21+
try self.runCodeGeneration(sourceDir: Paths.sourcesDir)
22+
}
23+
}

‎SwiftSyntaxDevUtils/Sources/swift-syntax-dev-utils/commands/Test.swift

Lines changed: 170 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,170 @@
1+
//===----------------------------------------------------------------------===//
2+
//
3+
// This source file is part of the Swift.org open source project
4+
//
5+
// Copyright (c) 2014 - 2023 Apple Inc. and the Swift project authors
6+
// Licensed under Apache License v2.0 with Runtime Library Exception
7+
//
8+
// See https://swift.org/LICENSE.txt for license information
9+
// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
10+
//
11+
//===----------------------------------------------------------------------===//
12+
13+
import ArgumentParser
14+
import Foundation
15+
16+
struct Test: ParsableCommand, Builder {
17+
@OptionGroup
18+
var arguments: BuilderArguments
19+
20+
@Flag(help: "Don't run lit-based tests")
21+
var skipLitTests: Bool = false
22+
23+
@Argument(
24+
help: """
25+
Path to the FileCheck executable that was built as part of the LLVM repository.
26+
If not specified, it will be looked up from PATH.
27+
"""
28+
)
29+
var filecheckExec: String?
30+
31+
func run() throws {
32+
try buildProduct(productName: "lit-test-helper")
33+
try buildExample(exampleName: "ExamplePlugin")
34+
35+
try runTests()
36+
37+
logSection("All tests passed")
38+
}
39+
40+
private func runTests() throws {
41+
logSection("Running SwiftSyntax Tests")
42+
43+
if !skipLitTests {
44+
try runLitTests()
45+
}
46+
47+
try runXctests()
48+
}
49+
50+
private func runLitTests() throws {
51+
logSection("Running lit-based tests")
52+
53+
guard FileManager.default.fileExists(atPath: Paths.litExec.path) else {
54+
throw ScriptExectutionError(
55+
message: """
56+
Error: Could not find lit.py.
57+
Looking at '\(Paths.litExec.path)'.
58+
59+
Make sure you have the llvm repo checked out next to the swift-syntax repo.
60+
Refer to README.md for more information.
61+
"""
62+
)
63+
}
64+
65+
guard FileManager.default.fileExists(atPath: Paths.incrTransferRoundtripExec.path) else {
66+
throw ScriptExectutionError(
67+
message: """
68+
Error: Could not find incr_transfer_round_trip.py.
69+
70+
Make sure you have the main swift repo checked out next to the swift-syntax
71+
repo.
72+
Refer to README.md for more information.
73+
"""
74+
)
75+
}
76+
77+
let productBinPath = try findProductBinPath()
78+
let examplesBinPath = try findExamplesBinPath()
79+
80+
let litTestHelperExec = productBinPath.appendingPathComponent("/lit-test-helper")
81+
var litCall = [
82+
Paths.litExec.path,
83+
Paths.packageDir.path,
84+
"lit_tests",
85+
]
86+
87+
if let filecheckExec {
88+
litCall += ["--param", "FILECHECK=" + filecheckExec]
89+
}
90+
91+
litCall += ["--param", "LIT_TEST_HELPER=" + litTestHelperExec.path]
92+
litCall += ["--param", "INCR_TRANSFER_ROUND_TRIP.PY=" + Paths.incrTransferRoundtripExec.path]
93+
94+
litCall += ["--param", "EXAMPLES_BIN_PATH=" + examplesBinPath.path]
95+
litCall += ["--param", "TOOLCHAIN=" + arguments.toolchain.path]
96+
97+
// Print all failures
98+
litCall += ["--verbose"]
99+
// Don't show all commands if verbose is not enabled
100+
if !arguments.verbose {
101+
litCall += ["--succinct"]
102+
}
103+
104+
guard let pythonExec = Paths.pythonExec else {
105+
throw ScriptExectutionError(message: "Didn't find python executable")
106+
}
107+
108+
let process = ProcessRunner(executableURL: pythonExec,
109+
arguments: litCall)
110+
111+
try process.run()
112+
}
113+
114+
private func runXctests() throws {
115+
logSection("Running XCTests")
116+
var swiftpmCallArguments: [String] = []
117+
118+
if arguments.verbose {
119+
swiftpmCallArguments += ["--verbose"]
120+
}
121+
122+
swiftpmCallArguments += ["--test-product", "SwiftSyntaxPackageTests"]
123+
124+
var additionalEnvironment: [String: String] = [:]
125+
additionalEnvironment["SWIFT_BUILD_SCRIPT_ENVIRONMENT"] = "1"
126+
127+
if arguments.enableRawsyntaxValidation {
128+
additionalEnvironment["SWIFTSYNTAX_ENABLE_RAWSYNTAX_VALIDATION"] = "1"
129+
}
130+
131+
if arguments.enableTestFuzzing {
132+
additionalEnvironment["SWIFTPARSER_ENABLE_ALTERNATE_TOKEN_INTROSPECTION"] = "1"
133+
}
134+
135+
// Tell other projects in the unified build to use local dependencies
136+
additionalEnvironment["SWIFTCI_USE_LOCAL_DEPS"] = "1"
137+
additionalEnvironment["SWIFT_SYNTAX_PARSER_LIB_SEARCH_PATH"] =
138+
arguments.toolchain
139+
.appendingPathComponent("lib")
140+
.appendingPathComponent("swift")
141+
.appendingPathComponent("macosx")
142+
.path
143+
144+
try invokeSwiftPM(
145+
action: "test",
146+
packageDir: Paths.packageDir,
147+
additionalArguments: swiftpmCallArguments,
148+
additionalEnvironment: additionalEnvironment
149+
)
150+
}
151+
152+
private func findSwiftpmBinPath(packageDir: URL) throws -> String {
153+
return try invokeSwiftPM(
154+
action: "build",
155+
packageDir: packageDir,
156+
additionalArguments: ["--show-bin-path"],
157+
additionalEnvironment: [:]
158+
)
159+
}
160+
161+
private func findProductBinPath() throws -> URL {
162+
let stdOut = try findSwiftpmBinPath(packageDir: Paths.packageDir)
163+
return URL(fileURLWithPath: stdOut)
164+
}
165+
166+
private func findExamplesBinPath() throws -> URL {
167+
let stdOut = try findSwiftpmBinPath(packageDir: Paths.examplesDir)
168+
return URL(fileURLWithPath: stdOut)
169+
}
170+
}

‎SwiftSyntaxDevUtils/Sources/swift-syntax-dev-utils/commands/VerifySourceCode.swift

Lines changed: 66 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,66 @@
1+
//===----------------------------------------------------------------------===//
2+
//
3+
// This source file is part of the Swift.org open source project
4+
//
5+
// Copyright (c) 2014 - 2023 Apple Inc. and the Swift project authors
6+
// Licensed under Apache License v2.0 with Runtime Library Exception
7+
//
8+
// See https://swift.org/LICENSE.txt for license information
9+
// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
10+
//
11+
//===----------------------------------------------------------------------===//
12+
13+
import ArgumentParser
14+
import Foundation
15+
16+
fileprivate var modules: [String] {
17+
["SwiftBasicFormat", "SwiftIDEUtils", "SwiftParser", "SwiftSyntax", "SwiftSyntaxBuilder"]
18+
}
19+
20+
struct VerifySourceCode: ParsableCommand, SourceCodeGeneratorCommand {
21+
@OptionGroup
22+
var arguments: SourceCodeGeneratorArguments
23+
24+
func run() throws {
25+
let tempDir = FileManager.default.temporaryDirectory
26+
27+
try self.runCodeGeneration(sourceDir: tempDir)
28+
29+
logSection("Verifing code generated files")
30+
31+
for module in modules {
32+
let selfGeneratedDir = tempDir.appendingPathComponent(module).appendingPathComponent("generated")
33+
let userGeneratedDir = Paths.sourcesDir.appendingPathComponent(module).appendingPathComponent("generated")
34+
try checkGeneratedFilesMatch(selfGeneratedDir: selfGeneratedDir, userGeneratedDir: userGeneratedDir)
35+
}
36+
}
37+
38+
private func checkGeneratedFilesMatch(
39+
selfGeneratedDir: URL,
40+
userGeneratedDir: URL
41+
) throws {
42+
43+
let selfGeneratedFiles = try FileManager.default.contentsOfDirectory(at: selfGeneratedDir, includingPropertiesForKeys: nil)
44+
let userGeneratedFiles = try FileManager.default.contentsOfDirectory(at: userGeneratedDir, includingPropertiesForKeys: nil)
45+
46+
try zip(selfGeneratedFiles, userGeneratedFiles)
47+
.forEach {
48+
if arguments.verbose {
49+
print("Comparing \($0.path) with \($1.path) ...")
50+
}
51+
52+
if !FileManager.default.contentsEqual(atPath: $0.path, andPath: $1.path) {
53+
throw ScriptExectutionError(
54+
message: """
55+
FAIL: code-generated files committed to repository do
56+
not match generated ones. Please re-generate the
57+
code-generated-files using the following command, open a PR to the
58+
SwiftSyntax project and merge it alongside the main PR.
59+
$ swift-syntax/build-script.py generate-source-code
60+
--toolchain /path/to/toolchain.xctoolchain/usr
61+
"""
62+
)
63+
}
64+
}
65+
}
66+
}

0 commit comments

Comments
 (0)