[Syntax] Use BumpPtrAllocator for Syntax node internals

Rework `Syntax` internals. "Red" tree is now bump-pointer allocated and
visited children are cached.

* `Syntax` is a pair of the allocator (strong reference to
  `SyntaxDataArena` class) and a pointer to the allocated data
  (`SyntaxData` struct).
* All the red-tree data (parent and positions) are
  bump-pointer-allocated and cached.
* The arena and the tree are 1:1. Each mutation creates a new arena.
* `child(at:)` is now O(1) because the red-tree children are cached in
  the arena.
* `root` is now O(1) regardless of the depth because the arena holds the
  reference.
* Simplify `SyntaxChildren`. `SyntaxChildrenIndex` is now just a `Int`

Remove some hacks as they aren't needed anymore:
* `UnownedSyntax` because `SyntaxDataReference` replaces it.
* `SyntaxNodeFactory` because `Syntax.Info` is gone.

Additionally:
* Flatten `AbsoluteSyntaxInfo` to simplify the data and clarify what
  `advancedBySibling(_:)` and `advancedToFirstChild()` do.
* Remove `RawSyntaxChildren` and `RawNonNilSyntaxChildren` as they were
  implementation detail of `SyntaxChildren`.
* Remove `AbsoluteRawSyntax` and `AbsoluteSyntaxPosition` as nobody was
  really using it.
* Rename `Syntax.indexInParent` to `layoutIndexInParent` because it was
  confusing with `SyntaxProtocol.indexInParent: SyntaxChildrenIndex`

Author: rintaro

CodeGeneration/Sources/generate-swift-syntax/templates/swiftsyntax/SyntaxRewriterFile.swift

@@ -42,13 +42,6 @@ let syntaxRewriterFile = SourceFileSyntax(leadingTrivia: copyrightHeader) {
       """
     )
 
-    DeclSyntax(
-      """
-      /// 'Syntax' object factory recycling 'Syntax.Info' instances.
-      private let nodeFactory: SyntaxNodeFactory = SyntaxNodeFactory()
-      """
-    )
-
     DeclSyntax(
       """
       public init(viewMode: SyntaxTreeViewMode = .sourceAccurate) {
@@ -322,13 +315,13 @@ let syntaxRewriterFile = SourceFileSyntax(leadingTrivia: copyrightHeader) {
         // with 'Syntax'
         var rewrittens: ContiguousArray<RetainedSyntaxArena> = []
 
-        for case let (child?, info) in RawSyntaxChildren(node) where viewMode.shouldTraverse(node: child) {
+        for case let childDataRef? in node.layoutBuffer where viewMode.shouldTraverse(node: childDataRef.pointee.raw) {
 
           // Build the Syntax node to rewrite
-          var childNode = nodeFactory.create(parent: node, raw: child, absoluteInfo: info)
+          var childNode = Syntax(arena: node.arena, dataRef: childDataRef)
 
           dispatchVisit(&childNode)
-          if childNode.raw.id != child.id {
+          if childNode.raw.id != childDataRef.pointee.raw.id {
             // The node was rewritten, let's handle it
 
             if newLayout.baseAddress == nil {
@@ -339,13 +332,10 @@ let syntaxRewriterFile = SourceFileSyntax(leadingTrivia: copyrightHeader) {
             }
 
             // Update the rewritten child.
-            newLayout[Int(info.indexInParent)] = childNode.raw
+            newLayout[Int(childDataRef.pointee.absoluteInfo.layoutIndexInParent)] = childNode.raw
             // Retain the syntax arena of the new node until it's wrapped with Syntax node.
             rewrittens.append(childNode.raw.arenaReference.retained)
           }
-
-          // Recycle 'childNode.info'
-          nodeFactory.dispose(&childNode)
         }
 
         if newLayout.baseAddress != nil {

CodeGeneration/Sources/generate-swift-syntax/templates/swiftsyntax/SyntaxVisitorFile.swift

@@ -32,13 +32,6 @@ let syntaxVisitorFile = SourceFileSyntax(leadingTrivia: copyrightHeader) {
   try! ClassDeclSyntax("open class SyntaxVisitor") {
     DeclSyntax("public let viewMode: SyntaxTreeViewMode")
 
-    DeclSyntax(
-      """
-      /// 'Syntax' object factory recycling 'Syntax.Info' instances.
-      private let nodeFactory: SyntaxNodeFactory = SyntaxNodeFactory()
-      """
-    )
-
     DeclSyntax(
       """
       public init(viewMode: SyntaxTreeViewMode) {
@@ -243,10 +236,9 @@ let syntaxVisitorFile = SourceFileSyntax(leadingTrivia: copyrightHeader) {
       """
       /// - Note: `node` is `inout` to avoid reference counting. See comment in `visitImpl`.
       private func visitChildren(_ node: inout Syntax) {
-        for case let (child?, info) in RawSyntaxChildren(node) where viewMode.shouldTraverse(node: child) {
-          var childNode = nodeFactory.create(parent: node, raw: child, absoluteInfo: info)
-          visit(&childNode)
-          nodeFactory.dispose(&childNode)
+        for case let childDataRef? in node.layoutBuffer where viewMode.shouldTraverse(node: childDataRef.pointee.raw) {
+          var child = Syntax(arena: node.arena, dataRef: childDataRef)
+          visit(&child)
         }
       }
       """

Release Notes/610.md

@@ -0,0 +1,26 @@
+# Swift Syntax 610 Release Notes
+
+## New APIs
+
+## API Behavior Changes
+
+## Deprecations
+
+## API-Incompatible Changes
+
+- `SyntaxChildrenIndex` is no longer `ExpressibleByNilLiteral`
+  - Description: `nil` used to represent the end index. However, due to a change in the internal structure, the end index must now be retrieved from the collection.
+  - Pull Request: https://github.com/swiftlang/swift-syntax/pull/2925
+  - Migration steps: Use `SyntaxChildrenIndex.endIndex` instead.
+  - Notes: `ExpressibleByNilLiteral` was a mistake. In general, `Collection.Index` should only be created and managed by the collection itself. For example, `Collection.index(after:)` exists, but `Index.advanced(by:)` does not.
+
+## Template
+
+- *Affected API or two word description*
+  - Description: *A 1-2 sentence description of the new/modified API*
+  - Issue: *If an issue exists for this change, a link to the issue*
+  - Pull Request: *Link to the pull request(s) that introduces this change*
+  - Migration steps: Steps that adopters of swift-syntax should take to move to the new API (required for deprecations and API-incompatible changes).
+  - Notes: *In case of deprecations or API-incompatible changes, the reason why this change was made and the suggested alternative*
+
+*Insert entries in chronological order, with newer entries at the bottom*

Sources/SwiftSyntax/AbsoluteRawSyntax.swift

@@ -2,57 +2,59 @@
 //
 // This source file is part of the Swift.org open source project
 //
-// Copyright (c) 2014 - 2023 Apple Inc. and the Swift project authors
+// Copyright (c) 2014 - 2024 Apple Inc. and the Swift project authors
 // Licensed under Apache License v2.0 with Runtime Library Exception
 //
 // See https://swift.org/LICENSE.txt for license information
 // See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
 //
 //===----------------------------------------------------------------------===//
 
-struct AbsoluteSyntaxPosition: Sendable {
-  /// The UTF-8 offset of the syntax node in the source file
-  let offset: UInt32
-  let indexInParent: UInt32
-
-  func advancedBySibling(_ raw: RawSyntax?) -> AbsoluteSyntaxPosition {
-    let newOffset = self.offset + UInt32(truncatingIfNeeded: raw?.totalLength.utf8Length ?? 0)
-    let newIndexInParent = self.indexInParent + 1
-    return .init(offset: newOffset, indexInParent: newIndexInParent)
-  }
-
-  func advancedToFirstChild() -> AbsoluteSyntaxPosition {
-    return .init(offset: self.offset, indexInParent: 0)
-  }
-
-  static var forRoot: AbsoluteSyntaxPosition {
-    return .init(offset: 0, indexInParent: 0)
-  }
-}
-
 /// `AbsoluteSyntaxInfo` represents the information that relates a `RawSyntax`
 /// to a source file tree, like its absolute source offset.
 struct AbsoluteSyntaxInfo: Sendable {
-  let position: AbsoluteSyntaxPosition
-  let nodeId: SyntaxIdentifier
+  /// Byte offset in the tree.
+  let offset: UInt32
+
+  /// Index in parent's layout. Note that this counts `nil` children.
+  let layoutIndexInParent: UInt32
 
-  /// The UTF-8 offset of the syntax node in the source file
-  var offset: UInt32 { return position.offset }
-  var indexInParent: UInt32 { return position.indexInParent }
+  /// index of the node when traversing the syntax tree using a depth-first traversal.
+  /// This skips `nil` children in the parent's layout.
+  let indexInTree: UInt32
 
   func advancedBySibling(_ raw: RawSyntax?) -> AbsoluteSyntaxInfo {
-    let newPosition = position.advancedBySibling(raw)
-    let newNodeId = nodeId.advancedBySibling(raw)
-    return .init(position: newPosition, nodeId: newNodeId)
+    if let raw {
+      return AbsoluteSyntaxInfo(
+        offset: offset &+ UInt32(truncatingIfNeeded: raw.totalLength.utf8Length),
+        layoutIndexInParent: layoutIndexInParent &+ 1,
+        indexInTree: indexInTree &+ UInt32(truncatingIfNeeded: raw.totalNodes)
+      )
+    } else {
+      return AbsoluteSyntaxInfo(
+        offset: offset,
+        layoutIndexInParent: layoutIndexInParent &+ 1,
+        indexInTree: indexInTree
+      )
+    }
   }
 
   func advancedToFirstChild() -> AbsoluteSyntaxInfo {
-    let newPosition = position.advancedToFirstChild()
-    let newNodeId = nodeId.advancedToFirstChild()
-    return .init(position: newPosition, nodeId: newNodeId)
+    return AbsoluteSyntaxInfo(
+      offset: offset,
+      layoutIndexInParent: 0,
+      indexInTree: indexInTree &+ 1
+    )
   }
 
   static func forRoot(_ raw: RawSyntax) -> AbsoluteSyntaxInfo {
-    return .init(position: .forRoot, nodeId: .forRoot(raw))
+    // These checks ensure the safety of the unchecked arithmetic operations in 'advancedBySibling(_:)'.
+    precondition(raw.totalLength.utf8Length <= UInt32.max, "too long")
+    precondition(raw.totalNodes <= UInt32.max, "too many nodes")
+    return AbsoluteSyntaxInfo(
+      offset: 0,
+      layoutIndexInParent: 0,
+      indexInTree: 0
+    )
   }
 }

Sources/SwiftSyntax/AbsoluteSyntaxInfo.swift

@@ -8,7 +8,6 @@
 
 add_swift_syntax_library(SwiftSyntax
   AbsolutePosition.swift
-  AbsoluteRawSyntax.swift
   AbsoluteSyntaxInfo.swift
   Assert.swift
   BumpPtrAllocator.swift
@@ -31,7 +30,6 @@ add_swift_syntax_library(SwiftSyntax
   SyntaxCollection.swift
   SyntaxHashable.swift
   SyntaxIdentifier.swift
-  SyntaxNodeFactory.swift
   SyntaxNodeStructure.swift
   SyntaxProtocol.swift
   SyntaxText.swift

Sources/SwiftSyntax/CMakeLists.txt

@@ -186,10 +186,10 @@ extension RawSyntaxData.ParsedToken {
 
 extension RawSyntaxData.MaterializedToken {
   var leadingTrivia: RawTriviaPieceBuffer {
-    triviaPieces[..<Int(numLeadingTrivia)]
+    RawTriviaPieceBuffer(rebasing: triviaPieces[..<Int(numLeadingTrivia)])
   }
   var trailingTrivia: RawTriviaPieceBuffer {
-    triviaPieces[Int(numLeadingTrivia)...]
+    RawTriviaPieceBuffer(rebasing: triviaPieces[Int(numLeadingTrivia)...])
   }
 }
 
@@ -215,7 +215,7 @@ public struct RawSyntax: Sendable {
   }
 
   var rawData: RawSyntaxData {
-    pointer.pointee
+    @_transparent unsafeAddress { pointer.pointer }
   }
 
   internal var arenaReference: SyntaxArenaRef {
@@ -954,7 +954,7 @@ extension RawSyntax {
 extension RawSyntax: Identifiable {
   public struct ID: Hashable, @unchecked Sendable {
     /// The pointer to the start of the `RawSyntax` node.
-    private var pointer: UnsafeRawPointer
+    fileprivate var pointer: UnsafeRawPointer
     fileprivate init(_ raw: RawSyntax) {
       self.pointer = raw.pointer.unsafeRawPointer
     }
@@ -965,6 +965,13 @@ extension RawSyntax: Identifiable {
   }
 }
 
+extension UInt {
+  /// Convert `RawSymtax.ID` to `UInt`. Lossless.
+  init(rawID: RawSyntax.ID) {
+    self.init(bitPattern: rawID.pointer)
+  }
+}
+
 /// See `SyntaxMemoryLayout`.
 let RawSyntaxDataMemoryLayouts: [String: SyntaxMemoryLayout.Value] = [
   "RawSyntaxData": .init(RawSyntaxData.self),