Re-architect format.py in swift

.gitignore

@@ -26,3 +26,6 @@ Package.resolved
 *.pyc
 
 Tests/PerformanceTest/baselines.json
+
+# The local build of swift-format to format swift-syntax
+.swift-format-build

CONTRIBUTING.md

@@ -21,12 +21,27 @@ swift-syntax is a SwiftPM package, so you can build and test it using anything t
 
 swift-syntax is formatted using [swift-format](http://github.com/apple/swift-format) to ensure a consistent style.
 
-To format your changes run `format.py` at the root of this repository. If you have a `swift-format` executable ready, you can pass it to `format.py`. If you do not, `format.py` will build its own copy of `swift-format` in `/tmp/swift-format`.
-
-If you are seeing surprising formatting results, you most likely have a `swift-format` installed on your system that’s not the most recent version built from the `main` branch. To fix this, clone [swift-format](http://github.com/apple/swift-format), build it using `swift build` and pass the freshly built executable to `format.py` as `--swift-format path/to/swift-format/.build/debug/swift-format`. Alternatively, you can uninstall `swift-format` on your system and `format.py` will build it from scratch.
+To format your changes run the formatter using the following command
+```bash
+swift run --package-path SwiftSyntaxDevUtils/ swift-syntax-dev-utils format
+```
+It will build a local copy of swift-format from the `main` branch and format the repository. Since it is building a release version of `swift-format`, the first run will take few minutes. Subsequent runs take less than 2 seconds.
 
 Generated source code is not formatted to make it easier to spot changes when re-running code generation.
 
+> [!NOTE]
+> You can add a git hook to ensure all commits to the swift-syntax repository are correctly formatted. 
+> 1. Save the following contents to `swift-syntax/.git/hooks/pre-commit`
+> ```bash
+> #!/usr/bin/env bash
+> set -e
+> SOURCE_DIR=$(realpath "$(dirname $0)/../..")
+> swift run --package-path "$SOURCE_DIR/SwiftSyntaxDevUtils" swift-syntax-dev-utils format --lint
+> ```
+> 2. Mark the file as executable: `chmod a+x swift-syntax/.git/hooks/pre-commit`
+> 3. If you have global git hooks installed, be sure to call them at the end of the script with `path/to/global/hooks/pre-commit "$@"` 
+
+
 ## Generating Source Code
 
 If you want to modify the generated files, open the [CodeGeneration](CodeGeneration) package and run the `generate-swift-syntax` executable.

SwiftSyntaxDevUtils/Sources/swift-syntax-dev-utils/SwiftSyntaxDevUtils.swift

@@ -26,6 +26,7 @@ struct SwiftSyntaxDevUtils: ParsableCommand {
       """,
     subcommands: [
       Build.self,
+      Format.self,
       GenerateSourceCode.self,
       Test.self,
       VerifySourceCode.self,

SwiftSyntaxDevUtils/Sources/swift-syntax-dev-utils/commands/Format.swift

@@ -0,0 +1,256 @@
+//===----------------------------------------------------------------------===//
+//
+// This source file is part of the Swift.org open source project
+//
+// Copyright (c) 2014 - 2023 Apple Inc. and the Swift project authors
+// Licensed under Apache License v2.0 with Runtime Library Exception
+//
+// See https://swift.org/LICENSE.txt for license information
+// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
+//
+//===----------------------------------------------------------------------===//
+
+import ArgumentParser
+import Foundation
+
+/// Directories that should not be formatted.
+fileprivate let directoriesToExclude = [
+  "lit_tests",
+  "generated",
+  "build",
+  "Inputs",
+  ".build",
+  ".swift-format-build",
+]
+
+struct Format: ParsableCommand {
+  static var configuration: CommandConfiguration {
+    return CommandConfiguration(
+      abstract: "Format files in SwiftSyntax using swift-format.",
+      discussion: """
+        This command automatically builds the '\(Self.swiftFormatBranch)' branch \
+        of swift-format in the '\(Paths.swiftFormatBuildDir.lastPathComponent)' \
+        directory of this repository and uses the build to format the swift-syntax \
+        sources.
+        """
+    )
+  }
+
+  @Flag(help: "Update the sources of swift-format and rebuild swift-format")
+  var update: Bool = false
+
+  @Flag(help: "Instead of formatting in-place, verify that the files are correctly formatted. Exit with 1 if files are not correctly formatted.")
+  var lint: Bool = false
+
+  @Option(
+    help: """
+      Instead of building a local swift-format, use this swift-format executable. \
+      Should primarily be used for CI, which has already built swift-format.
+      """
+  )
+  var swiftFormat: String? = nil
+
+  @Flag(help: "Enable verbose logging.")
+  var verbose: Bool = false
+
+  /// The configuration to build swift-format in.
+  private static let swiftFormatBuildConfiguration: String = "release"
+
+  /// The branch of swift-format to build.
+  private static let swiftFormatBranch: String = "main"
+
+  private enum Error: Swift.Error, CustomStringConvertible {
+    case swiftFormatNotFound
+    case lintFailed
+
+    var description: String {
+      switch self {
+      case .swiftFormatNotFound:
+        return "The locally built swift-format could not be found"
+      case .lintFailed:
+        return """
+          The swift-syntax repo is not formatted according to the style guides.
+
+          Run the following command to format swift-syntax
+
+            swift run --package-path SwiftSyntaxDevUtils/ swift-syntax-dev-utils format
+
+          If the issue persists, try updating swift-format by running
+
+            swift run --package-path SwiftSyntaxDevUtils/ swift-syntax-dev-utils format --update
+
+          Should that still fail, fix any remaining issues manually and verify they match the swift-format style using
+
+            swift run --package-path SwiftSyntaxDevUtils/ swift-syntax-dev-utils format --lint
+          """
+      }
+    }
+  }
+
+  /// Run `git` in the .swift-format-build directory with the provided arguments.
+  private func runGitCommand(_ arguments: String...) throws {
+    try ProcessRunner(
+      executableURL: Paths.gitExec,
+      arguments: ["-C", Paths.swiftFormatBuildDir.path] + arguments
+    ).run(
+      captureStdout: false,
+      captureStderr: false,
+      verbose: verbose
+    )
+  }
+
+  /// Run `swift` for the `.swift-format-build` package with the provided arguments.
+  private func runSwiftCommand(_ action: String, _ arguments: String...) throws {
+    try ProcessRunner(
+      executableURL: Paths.swiftExec,
+      arguments: [action, "--package-path", Paths.swiftFormatBuildDir.path] + arguments
+    ).run(
+      captureStdout: false,
+      captureStderr: false,
+      verbose: verbose
+    )
+  }
+
+  /// Ensure that we have an up-to-date checkout of swift-format in `.swift-format-build`.
+  private func cloneOrUpdateSwiftFormat() throws {
+    if FileManager.default.fileExists(atPath: Paths.swiftFormatBuildDir.appendingPathComponent(".git").path) {
+      try runGitCommand("checkout", Self.swiftFormatBranch)
+      try runGitCommand("pull")
+    } else {
+      try FileManager.default.createDirectory(atPath: Paths.swiftFormatBuildDir.path, withIntermediateDirectories: true)
+      try runGitCommand("clone", "https://github.com/apple/swift-format.git", ".")
+      try runGitCommand("checkout", Self.swiftFormatBranch)
+    }
+    try runSwiftCommand("package", "update")
+  }
+
+  /// Build the swift-format executable.
+  private func buildSwiftFormat() throws {
+    try runSwiftCommand("build", "--product", "swift-format", "--configuration", Self.swiftFormatBuildConfiguration)
+  }
+
+  /// Get the URL of the locally-built swift-format executable.
+  private func findSwiftFormatExecutable() throws -> URL {
+    if let swiftFormat = swiftFormat {
+      return URL(fileURLWithPath: swiftFormat)
+    }
+
+    // We could run `swift build --show-bin-path` here but that takes 0.4s.
+    // Since the path seems really stable, let’s build the path ourselves.
+    let swiftFormatExec = URL(fileURLWithPath: Paths.swiftFormatBuildDir.path)
+      .appendingPathComponent(".build")
+      .appendingPathComponent(Self.swiftFormatBuildConfiguration)
+      .appendingPathComponent("swift-format")
+    if !swiftFormatExec.isExecutableFile {
+      throw Error.swiftFormatNotFound
+    }
+    return swiftFormatExec
+  }
+
+  /// Get the list of files that should be formatted using swift-format.
+  ///
+  /// This excludes some files like generated files or test inputs.
+  private func filesToFormat() -> [URL] {
+    guard let enumerator = FileManager.default.enumerator(at: Paths.packageDir.resolvingSymlinksInPath(), includingPropertiesForKeys: [], options: []) else {
+      return []
+    }
+
+    var result: [URL] = []
+    for case let url as URL in enumerator {
+      if directoriesToExclude.contains(url.lastPathComponent) {
+        enumerator.skipDescendants()
+      }
+      if url.pathExtension == "swift" {
+        result.append(url)
+      }
+    }
+
+    return result
+  }
+
+  /// Format all files in the repo using the locally-built swift-format.
+  private func formatFilesInRepo() throws {
+    let swiftFormatExecutable = try findSwiftFormatExecutable()
+
+    let filesToFormat = self.filesToFormat()
+
+    try ProcessRunner(
+      executableURL: swiftFormatExecutable,
+      arguments: [
+        "format",
+        "--in-place",
+        "--parallel",
+      ] + filesToFormat.map { $0.path }
+    )
+    .run(
+      captureStdout: false,
+      captureStderr: false,
+      verbose: verbose
+    )
+  }
+
+  /// Lint all files in the repo using the locally-built swift-format.
+  private func lintFilesInRepo() throws {
+    let swiftFormatExecutable = try findSwiftFormatExecutable()
+
+    let filesToFormat = self.filesToFormat()
+
+    do {
+      try ProcessRunner(
+        executableURL: swiftFormatExecutable,
+        arguments: [
+          "lint",
+          "--strict",
+          "--parallel",
+        ] + filesToFormat.map { $0.path }
+      )
+      .run(
+        captureStdout: false,
+        captureStderr: false,
+        verbose: verbose
+      )
+    } catch is NonZeroExitCodeError {
+      throw Error.lintFailed
+    }
+  }
+
+  func run() throws {
+    #if compiler(<5.10)
+    print("💡 You are building running the format script with Swift 5.9 or lower. Running it with SwiftPM 5.10 is about 10s faster.")
+    #endif
+
+    try run(updateAndBuild: self.update)
+  }
+
+  /// - Parameter updateAndBuild: Whether to update the locally checked out
+  ///   swift-format sources and rebuild swift-format.
+  func run(updateAndBuild: Bool) throws {
+    if updateAndBuild {
+      try cloneOrUpdateSwiftFormat()
+      try buildSwiftFormat()
+    }
+    do {
+      if lint {
+        try lintFilesInRepo()
+      } else {
+        try formatFilesInRepo()
+      }
+    } catch Error.swiftFormatNotFound {
+      if !updateAndBuild {
+        print(
+          """
+          No build of swift-format was found in '\(Paths.swiftFormatBuildDir.lastPathComponent)'.
+          Building swift-format now. This may take a couple of minutes.
+          Future invocations of this command will re-use the build and are much faster.
+
+          """
+        )
+
+        // If swift-format cannot be found, try again, updating (aka cloning + building) swift-format this time
+        try run(updateAndBuild: true)
+      } else {
+        throw Error.swiftFormatNotFound
+      }
+    }
+  }
+}

SwiftSyntaxDevUtils/Sources/swift-syntax-dev-utils/commands/VerifySourceCode.swift

@@ -36,16 +36,12 @@ struct VerifySourceCode: ParsableCommand, SourceCodeGeneratorCommand {
 
     logSection("Verifing code generated files")
 
-    guard let diffExec = Paths.diffExec else {
-      throw ScriptExectutionError(message: "Didn't find a diff execution path")
-    }
-
     for module in modules {
       let selfGeneratedDir = tempDir.appendingPathComponent(module).appendingPathComponent("generated")
       let userGeneratedDir = Paths.sourcesDir.appendingPathComponent(module).appendingPathComponent("generated")
 
       let process = ProcessRunner(
-        executableURL: diffExec,
+        executableURL: try Paths.diffExec,
         arguments: [
           "--recursive",
           "--exclude",

SwiftSyntaxDevUtils/Sources/swift-syntax-dev-utils/common/BuildCommand.swift

@@ -86,16 +86,8 @@ extension BuildCommand {
   @discardableResult
   func invokeXcodeBuild(projectPath: URL, scheme: String) throws -> ProcessResult {
     return try withTemporaryDirectory { tempDir in
-      guard let xcodebuildExec = Paths.xcodebuildExec else {
-        throw ScriptExectutionError(
-          message: """
-            Error: Could not find xcodebuild.
-            Looking at '\(Paths.xcodebuildExec?.path ?? "N/A")'.
-            """
-        )
-      }
       let processRunner = ProcessRunner(
-        executableURL: xcodebuildExec,
+        executableURL: try Paths.xcodebuildExec,
         arguments: [
           "-project", projectPath.path,
           "-scheme", scheme,