[Syntax] Use BumpPtrAllocator for Syntax node internals

Rework 'Syntax' internals. "Red" tree are now bump-pointer allocated and
the visited chilren are cached.

* 'Syntax' is a pair of the allocator (retaing ref to 'SyntaxDataArena'
class) and a pointer to the allocated data ('SyntaxData' struct).
* 'child(at:)' is now O(n) because the red-tree children are cached.
* Simplifed 'SyntaxChildren'. 'SyntaxChildrenIndex' is now just a 'Int'

Remove some hacks as they aren't needed anymore:
* 'UnownedSyntax' because 'SyntaxDataReference' replaces it.
* 'SyntaxNodeFactory' because 'Syntax.Info' is gone.

Additionally:
* Flatten 'AbsoluteSyntaxInfo' to simplify the data and clarify what
  'advancedBySibling(_:)' and 'advancedToFirstChild()' do.
* Remove 'RawSyntaxChildren' and 'RawNonNilSyntaxChildren' as they were
  implementation detail of 'SyntaxChildren'.
* Remove 'AbsoluteRawSyntax' and 'AbsoluteSyntaxPosition' as nobody
  was really using it.
* Rename 'Syntax.indexInParent' to 'layoutIndexInParent' because it was
  confusing with 'SyntaxProtocol.indexInParent: SyntaxChildrenIndex'

Author: rintaro

CodeGeneration/Sources/generate-swift-syntax/templates/swiftsyntax/SyntaxRewriterFile.swift

@@ -42,13 +42,6 @@ let syntaxRewriterFile = SourceFileSyntax(leadingTrivia: copyrightHeader) {
       """
     )
 
-    DeclSyntax(
-      """
-      /// 'Syntax' object factory recycling 'Syntax.Info' instances.
-      private let nodeFactory: SyntaxNodeFactory = SyntaxNodeFactory()
-      """
-    )
-
     DeclSyntax(
       """
       public init(viewMode: SyntaxTreeViewMode = .sourceAccurate) {
@@ -322,13 +315,13 @@ let syntaxRewriterFile = SourceFileSyntax(leadingTrivia: copyrightHeader) {
         // with 'Syntax'
         var rewrittens: ContiguousArray<RetainedSyntaxArena> = []
 
-        for case let (child?, info) in RawSyntaxChildren(node) where viewMode.shouldTraverse(node: child) {
+        for case let childDataRef? in node.layoutBuffer where viewMode.shouldTraverse(node: childDataRef.pointee.raw) {
 
           // Build the Syntax node to rewrite
-          var childNode = nodeFactory.create(parent: node, raw: child, absoluteInfo: info)
+          var childNode = Syntax(arena: node.arena, dataRef: childDataRef)
 
           dispatchVisit(&childNode)
-          if childNode.raw.id != child.id {
+          if childNode.raw.id != childDataRef.pointee.raw.id {
             // The node was rewritten, let's handle it
 
             if newLayout.baseAddress == nil {
@@ -339,13 +332,10 @@ let syntaxRewriterFile = SourceFileSyntax(leadingTrivia: copyrightHeader) {
             }
 
             // Update the rewritten child.
-            newLayout[Int(info.indexInParent)] = childNode.raw
+            newLayout[Int(childDataRef.pointee.absoluteInfo.layoutIndexInParent)] = childNode.raw
             // Retain the syntax arena of the new node until it's wrapped with Syntax node.
             rewrittens.append(childNode.raw.arenaReference.retained)
           }
-
-          // Recycle 'childNode.info'
-          nodeFactory.dispose(&childNode)
         }
 
         if newLayout.baseAddress != nil {

CodeGeneration/Sources/generate-swift-syntax/templates/swiftsyntax/SyntaxVisitorFile.swift

@@ -32,13 +32,6 @@ let syntaxVisitorFile = SourceFileSyntax(leadingTrivia: copyrightHeader) {
   try! ClassDeclSyntax("open class SyntaxVisitor") {
     DeclSyntax("public let viewMode: SyntaxTreeViewMode")
 
-    DeclSyntax(
-      """
-      /// 'Syntax' object factory recycling 'Syntax.Info' instances.
-      private let nodeFactory: SyntaxNodeFactory = SyntaxNodeFactory()
-      """
-    )
-
     DeclSyntax(
       """
       public init(viewMode: SyntaxTreeViewMode) {
@@ -243,10 +236,9 @@ let syntaxVisitorFile = SourceFileSyntax(leadingTrivia: copyrightHeader) {
       """
       /// - Note: `node` is `inout` to avoid reference counting. See comment in `visitImpl`.
       private func visitChildren(_ node: inout Syntax) {
-        for case let (child?, info) in RawSyntaxChildren(node) where viewMode.shouldTraverse(node: child) {
-          var childNode = nodeFactory.create(parent: node, raw: child, absoluteInfo: info)
-          visit(&childNode)
-          nodeFactory.dispose(&childNode)
+        for case let childDataRef? in node.layoutBuffer where viewMode.shouldTraverse(node: childDataRef.pointee.raw) {
+          var child = Syntax(arena: node.arena, dataRef: childDataRef)
+          visit(&child)
         }
       }
       """

Sources/SwiftSyntax/AbsoluteRawSyntax.swift

@@ -2,57 +2,59 @@
 //
 // This source file is part of the Swift.org open source project
 //
-// Copyright (c) 2014 - 2023 Apple Inc. and the Swift project authors
+// Copyright (c) 2014 - 2024 Apple Inc. and the Swift project authors
 // Licensed under Apache License v2.0 with Runtime Library Exception
 //
 // See https://swift.org/LICENSE.txt for license information
 // See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
 //
 //===----------------------------------------------------------------------===//
 
-struct AbsoluteSyntaxPosition: Sendable {
-  /// The UTF-8 offset of the syntax node in the source file
-  let offset: UInt32
-  let indexInParent: UInt32
-
-  func advancedBySibling(_ raw: RawSyntax?) -> AbsoluteSyntaxPosition {
-    let newOffset = self.offset + UInt32(truncatingIfNeeded: raw?.totalLength.utf8Length ?? 0)
-    let newIndexInParent = self.indexInParent + 1
-    return .init(offset: newOffset, indexInParent: newIndexInParent)
-  }
-
-  func advancedToFirstChild() -> AbsoluteSyntaxPosition {
-    return .init(offset: self.offset, indexInParent: 0)
-  }
-
-  static var forRoot: AbsoluteSyntaxPosition {
-    return .init(offset: 0, indexInParent: 0)
-  }
-}
-
 /// `AbsoluteSyntaxInfo` represents the information that relates a `RawSyntax`
 /// to a source file tree, like its absolute source offset.
 struct AbsoluteSyntaxInfo: Sendable {
-  let position: AbsoluteSyntaxPosition
-  let nodeId: SyntaxIdentifier
+  /// Byte offset in the tree.
+  let offset: UInt32
+
+  /// Index in parent's layout. Note that this counts `nil` children.
+  let layoutIndexInParent: UInt32
 
-  /// The UTF-8 offset of the syntax node in the source file
-  var offset: UInt32 { return position.offset }
-  var indexInParent: UInt32 { return position.indexInParent }
+  /// index of the node when traversing the syntax tree using a depth-first traversal.
+  /// This skips `nil` children in the parent's layout.
+  let indexInTree: UInt32
 
   func advancedBySibling(_ raw: RawSyntax?) -> AbsoluteSyntaxInfo {
-    let newPosition = position.advancedBySibling(raw)
-    let newNodeId = nodeId.advancedBySibling(raw)
-    return .init(position: newPosition, nodeId: newNodeId)
+    if let raw {
+      return AbsoluteSyntaxInfo(
+        offset: offset &+ UInt32(truncatingIfNeeded: raw.totalLength.utf8Length),
+        layoutIndexInParent: layoutIndexInParent &+ 1,
+        indexInTree: indexInTree &+ UInt32(truncatingIfNeeded: raw.totalNodes)
+      )
+    } else {
+      return AbsoluteSyntaxInfo(
+        offset: offset,
+        layoutIndexInParent: layoutIndexInParent &+ 1,
+        indexInTree: indexInTree
+      )
+    }
   }
 
   func advancedToFirstChild() -> AbsoluteSyntaxInfo {
-    let newPosition = position.advancedToFirstChild()
-    let newNodeId = nodeId.advancedToFirstChild()
-    return .init(position: newPosition, nodeId: newNodeId)
+    return AbsoluteSyntaxInfo(
+      offset: offset,
+      layoutIndexInParent: 0,
+      indexInTree: indexInTree &+ 1
+    )
   }
 
   static func forRoot(_ raw: RawSyntax) -> AbsoluteSyntaxInfo {
-    return .init(position: .forRoot, nodeId: .forRoot(raw))
+    // These checks ensure the safety of the unchecked arithmetic operations in 'advancedBySibling(_:)'.
+    precondition(raw.totalLength.utf8Length <= UInt32.max, "too long")
+    precondition(raw.totalNodes <= UInt32.max, "too many nodes")
+    return AbsoluteSyntaxInfo(
+      offset: 0,
+      layoutIndexInParent: 0,
+      indexInTree: 0
+    )
   }
 }

Sources/SwiftSyntax/AbsoluteSyntaxInfo.swift

@@ -8,7 +8,6 @@
 
 add_swift_syntax_library(SwiftSyntax
   AbsolutePosition.swift
-  AbsoluteRawSyntax.swift
   AbsoluteSyntaxInfo.swift
   Assert.swift
   BumpPtrAllocator.swift
@@ -19,6 +18,7 @@ add_swift_syntax_library(SwiftSyntax
   Identifier.swift
   MemoryLayout.swift
   MissingNodeInitializers.swift
+  PlatformMutex.swift
   SourceEdit.swift
   SourceLength.swift
   SourceLocation.swift
@@ -31,7 +31,6 @@ add_swift_syntax_library(SwiftSyntax
   SyntaxCollection.swift
   SyntaxHashable.swift
   SyntaxIdentifier.swift
-  SyntaxNodeFactory.swift
   SyntaxNodeStructure.swift
   SyntaxProtocol.swift
   SyntaxText.swift

Sources/SwiftSyntax/CMakeLists.txt

@@ -0,0 +1,125 @@
+//===----------------------------------------------------------------------===//
+//
+// This source file is part of the Swift.org open source project
+//
+// Copyright (c) 2022-2024 Apple Inc. and the Swift project authors
+// Licensed under Apache License v2.0 with Runtime Library Exception
+//
+// See https://swift.org/LICENSE.txt for license information
+// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
+//
+//===----------------------------------------------------------------------===//
+
+#if compiler(>=6.0)
+#if canImport(Darwin)
+private import Darwin
+#elseif canImport(Glibc)
+private import Glibc
+#elseif canImport(Bionic)
+private import Bionic
+#elseif canImport(Musl)
+private import Musl
+#elseif canImport(WinSDK)
+private import WinSDK
+#endif
+#else
+#if canImport(Darwin)
+import Darwin
+#elseif canImport(Glibc)
+import Glibc
+#elseif canImport(Bionic)
+import Bionic
+#elseif canImport(Musl)
+import Musl
+#elseif canImport(WinSDK)
+import WinSDK
+#endif
+#endif
+
+/// A protocol that platform-specific mutual exclusion primitives should conform to.
+final class PlatformMutex {
+  private typealias PlatformLock = UnsafeMutablePointer<Primitive>
+  private let lock: PlatformLock
+
+  /// Allocate memory for, and initialize, the mutex in a platform-specific fashion.
+  init() {
+    let storage = PlatformLock.allocate(capacity: 1)
+    storage.initialize(to: Primitive())
+    self.lock = storage
+    Self.initialize(self.lock)
+  }
+
+  /// Deinitialize and deallocate the memory associated with the mutex.
+  deinit {
+    Self.deinitialize(self.lock)
+    self.lock.deinitialize(count: 1)
+    self.lock.deallocate()
+  }
+
+  /// Invoke the provided block under the protection of the mutex.
+  func withGuard<T>(body: () throws -> T) rethrows -> T {
+    Self.lock(self.lock)
+    defer { Self.unlock(self.lock) }
+    return try body()
+  }
+}
+
+#if canImport(Darwin)
+extension PlatformMutex {
+  private typealias Primitive = os_unfair_lock
+
+  private static func initialize(_ platformLock: PlatformLock) {
+    // Platform specific initialization is not needed.
+  }
+  private static func deinitialize(_ platformLock: PlatformLock) {
+    // Platform specific deinitialization is not needed.
+  }
+  private static func lock(_ platformLock: PlatformLock) {
+    os_unfair_lock_lock(platformLock)
+  }
+  private static func unlock(_ platformLock: PlatformLock) {
+    os_unfair_lock_unlock(platformLock)
+  }
+}
+
+#elseif canImport(Glibc) || canImport(Bionic) || canImport(Musl)
+extension PlatformMutex {
+  private typealias Primitive = pthread_mutex_t
+
+  private static func initialize(_ platformLock: PlatformLock) {
+    let result = pthread_mutex_init(platformLock, nil)
+    precondition(result == 0)
+  }
+  private static func deinitialize(_ platformLock: PlatformLock) {
+    let result = pthread_mutex_destroy(self.lock)
+    precondition(result == 0)
+  }
+  private static func lock(_ platformLock: PlatformLock) {
+    let result = pthread_mutex_lock(platformLock)
+    assert(result == 0)
+  }
+  private static func unlock(_ platformLock: PlatformLock) {
+    let result = pthread_mutex_unlock(platformLock)
+    assert(result == 0)
+  }
+}
+
+#elseif canImport(WinSDK)
+extension PlatformMutex {
+  private typealias Primitive = SRWLOCK
+
+  private static func initialize(_ platformLock: PlatformLock) {
+    InitializeSRWLock(platformLock)
+  }
+  private static func deinitialize(_ platformLock: PlatformLock) {
+    // Platform specific deinitialization is not needed.
+  }
+  private static func lock(_ platformLock: PlatformLock) {
+    AcquireSRWLockExclusive(platformLock)
+  }
+  private static func unlock(_ platformLock: PlatformLock) {
+    ReleaseSRWLockExclusive(platformLock)
+  }
+}
+
+#endif