Teach TokenSpecSet about experimental features

CodeGeneration/Sources/SyntaxSupport/ExperimentalFeatures.swift

@@ -0,0 +1,33 @@
+//===----------------------------------------------------------------------===//
+//
+// This source file is part of the Swift.org open source project
+//
+// Copyright (c) 2023 Apple Inc. and the Swift project authors
+// Licensed under Apache License v2.0 with Runtime Library Exception
+//
+// See https://swift.org/LICENSE.txt for license information
+// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
+//
+//===----------------------------------------------------------------------===//
+
+import SwiftSyntax
+
+public enum ExperimentalFeature: String, CaseIterable {
+  case referenceBindings
+  case thenStatements
+
+  /// The name of the feature, which is used in the doc comment.
+  public var featureName: String {
+    switch self {
+    case .referenceBindings:
+      return "reference bindings"
+    case .thenStatements:
+      return "'then' statements"
+    }
+  }
+
+  /// The token that represents the experimental feature case name.
+  public var token: TokenSyntax {
+    .identifier(rawValue)
+  }
+}

CodeGeneration/Sources/SyntaxSupport/KeywordSpec.swift

@@ -16,14 +16,18 @@ public struct KeywordSpec {
   /// The name of the keyword.
   public let name: String
 
+  /// The experimental feature the keyword is part of, or `nil` if this isn't
+  /// for an experimental feature.
+  public let experimentalFeature: ExperimentalFeature?
+
+  /// Indicates if the token kind is switched from being an identifier to a keyword in the lexer.
+  public let isLexerClassified: Bool
+
   /// Indicates if the keyword is part of an experimental language feature.
   ///
   /// If `true`, this keyword is for an experimental language feature, and any public
   /// API generated should be marked as SPI
-  public let isExperimental: Bool
-
-  /// Indicates if the token kind is switched from being an identifier to a keyword in the lexer.
-  public let isLexerClassified: Bool
+  public var isExperimental: Bool { experimentalFeature != nil }
 
   /// The name of this keyword that's suitable to be used for variable or enum case names.
   public var varOrCaseName: TokenSyntax {
@@ -46,15 +50,15 @@ public struct KeywordSpec {
   ///
   /// - Parameters:
   ///   - name: A name of the keyword.
-  ///   - isExperimental: Indicates if the keyword is part of an experimental language feature.
+  ///   - experimentalFeature: The experimental feature the keyword is part of, or `nil` if this isn't for an experimental feature.
   ///   - isLexerClassified: Indicates if the token kind is switched from being an identifier to a keyword in the lexer.
   init(
     _ name: String,
-    isExperimental: Bool = false,
+    experimentalFeature: ExperimentalFeature? = nil,
     isLexerClassified: Bool = false
   ) {
     self.name = name
-    self.isExperimental = isExperimental
+    self.experimentalFeature = experimentalFeature
     self.isLexerClassified = isLexerClassified
   }
 }
@@ -692,11 +696,11 @@ public enum Keyword: CaseIterable {
     case .yield:
       return KeywordSpec("yield")
     case ._borrowing:
-      return KeywordSpec("_borrowing", isExperimental: true)
+      return KeywordSpec("_borrowing", experimentalFeature: .referenceBindings)
     case ._consuming:
-      return KeywordSpec("_consuming", isExperimental: true)
+      return KeywordSpec("_consuming", experimentalFeature: .referenceBindings)
     case ._mutating:
-      return KeywordSpec("_mutating", isExperimental: true)
+      return KeywordSpec("_mutating", experimentalFeature: .referenceBindings)
     }
   }
 }

CodeGeneration/Sources/SyntaxSupport/Node.swift

@@ -40,9 +40,9 @@ public class Node {
   /// The kind of node’s supertype. This kind must have `isBase == true`
   public let base: SyntaxNodeKind
 
-  /// If `true`, this is for an experimental language feature, and any public
-  /// API generated should be SPI.
-  public let isExperimental: Bool
+  /// The experimental feature the node is part of, or `nil` if this isn't
+  /// for an experimental feature.
+  public let experimentalFeature: ExperimentalFeature?
 
   /// When the node name is printed for diagnostics, this name is used.
   /// If `nil`, `nameForDiagnostics` will print the parent node’s name.
@@ -57,6 +57,10 @@ public class Node {
   /// function that should be invoked to create this node.
   public let parserFunction: TokenSyntax?
 
+  /// If `true`, this is for an experimental language feature, and any public
+  /// API generated should be SPI.
+  public var isExperimental: Bool { experimentalFeature != nil }
+
   /// A name for this node that is suitable to be used as a variables or enum
   /// case's name.
   public var varOrCaseName: TokenSyntax {
@@ -107,7 +111,7 @@ public class Node {
   init(
     kind: SyntaxNodeKind,
     base: SyntaxNodeKind,
-    isExperimental: Bool = false,
+    experimentalFeature: ExperimentalFeature? = nil,
     nameForDiagnostics: String?,
     documentation: String? = nil,
     parserFunction: TokenSyntax? = nil,
@@ -119,7 +123,7 @@ public class Node {
 
     self.kind = kind
     self.base = base
-    self.isExperimental = isExperimental
+    self.experimentalFeature = experimentalFeature
     self.nameForDiagnostics = nameForDiagnostics
     self.documentation = docCommentTrivia(from: documentation)
     self.parserFunction = parserFunction
@@ -226,7 +230,7 @@ public class Node {
   init(
     kind: SyntaxNodeKind,
     base: SyntaxNodeKind,
-    isExperimental: Bool = false,
+    experimentalFeature: ExperimentalFeature? = nil,
     nameForDiagnostics: String?,
     documentation: String? = nil,
     parserFunction: TokenSyntax? = nil,
@@ -235,7 +239,7 @@ public class Node {
     self.kind = kind
     precondition(base == .syntaxCollection)
     self.base = base
-    self.isExperimental = isExperimental
+    self.experimentalFeature = experimentalFeature
     self.nameForDiagnostics = nameForDiagnostics
     self.documentation = docCommentTrivia(from: documentation)
     self.parserFunction = parserFunction

CodeGeneration/Sources/SyntaxSupport/StmtNodes.swift

@@ -588,7 +588,7 @@ public let STMT_NODES: [Node] = [
     kind: .thenStmt,
     base: .stmt,
     // FIXME: This should be marked experimental.
-    isExperimental: false,
+    experimentalFeature: nil,
     nameForDiagnostics: "'then' statement",
     documentation: """
       A statement used to indicate the produced value from an if/switch

CodeGeneration/Sources/SyntaxSupport/TokenSpec.swift

@@ -24,11 +24,9 @@ public struct TokenSpec {
   /// The name of the token, suitable for use in variable or enum case names.
   public let varOrCaseName: TokenSyntax
 
-  /// Indicates if the token is part of an experimental language feature.
-  ///
-  /// If `true`, this keyword is for an experimental language feature, and any public
-  /// API generated should be marked as SPI
-  public let isExperimental: Bool
+  /// The experimental feature the token is part of, or `nil` if this isn't
+  /// for an experimental feature.
+  public let experimentalFeature: ExperimentalFeature?
 
   /// The name of the token that can be shown in diagnostics.
   public let nameForDiagnostics: String
@@ -39,6 +37,12 @@ public struct TokenSpec {
   /// The kind of the token.
   public let kind: Kind
 
+  /// Indicates if the token is part of an experimental language feature.
+  ///
+  /// If `true`, this keyword is for an experimental language feature, and any public
+  /// API generated should be marked as SPI
+  public var isExperimental: Bool { experimentalFeature != nil }
+
   /// The attributes that should be printed on any API for the generated keyword.
   ///
   /// This is typically used to mark APIs as SPI when the keyword is part of an experimental language feature.
@@ -51,19 +55,19 @@ public struct TokenSpec {
   ///
   /// - Parameters:
   ///   - name: A name of the token.
-  ///   - isExperimental: Indicates if the token is part of an experimental language feature.
+  ///   - experimentalFeature: The experimental feature the token is part of, or `nil` if this isn't for an experimental feature.
   ///   - nameForDiagnostics: A name of the token that can be shown in diagnostics.
   ///   - text: An actual text of the token, if available.
   ///   - kind: A kind of the token.
   fileprivate init(
     name: String,
-    isExperimental: Bool = false,
+    experimentalFeature: ExperimentalFeature? = nil,
     nameForDiagnostics: String,
     text: String? = nil,
     kind: Kind
   ) {
     self.varOrCaseName = .identifier(name)
-    self.isExperimental = isExperimental
+    self.experimentalFeature = experimentalFeature
     self.nameForDiagnostics = nameForDiagnostics
     self.text = text
     self.kind = kind

CodeGeneration/Sources/generate-swift-syntax/GenerateSwiftSyntax.swift

@@ -88,6 +88,7 @@ struct GenerateSwiftSyntax: ParsableCommand {
 
     var fileSpecs: [GeneratedFileSpec] = [
       // SwiftParser
+      GeneratedFileSpec(swiftParserGeneratedDir + ["ExperimentalFeatures.swift"], experimentalFeaturesFile),
       GeneratedFileSpec(swiftParserGeneratedDir + ["IsLexerClassified.swift"], isLexerClassifiedFile),
       GeneratedFileSpec(swiftParserGeneratedDir + ["LayoutNodes+Parsable.swift"], layoutNodesParsableFile),
       GeneratedFileSpec(swiftParserGeneratedDir + ["Parser+TokenSpecSet.swift"], parserTokenSpecSetFile),

CodeGeneration/Sources/generate-swift-syntax/templates/swiftparser/ExperimentalFeaturesFile.swift

@@ -0,0 +1,43 @@
+//===----------------------------------------------------------------------===//
+//
+// This source file is part of the Swift.org open source project
+//
+// Copyright (c) 2023 Apple Inc. and the Swift project authors
+// Licensed under Apache License v2.0 with Runtime Library Exception
+//
+// See https://swift.org/LICENSE.txt for license information
+// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
+//
+//===----------------------------------------------------------------------===//
+
+import SwiftSyntax
+import SwiftSyntaxBuilder
+import SyntaxSupport
+import Utils
+
+let experimentalFeaturesFile = SourceFileSyntax(leadingTrivia: copyrightHeader) {
+  DeclSyntax(
+    """
+    extension Parser {
+      @_spi(ExperimentalLanguageFeatures)
+      public struct ExperimentalFeatures: OptionSet {
+        public let rawValue: UInt
+        public init(rawValue: UInt) {
+          self.rawValue = rawValue
+        }
+      }
+    }
+    """
+  )
+
+  try! ExtensionDeclSyntax("extension Parser.ExperimentalFeatures") {
+    for (i, feature) in ExperimentalFeature.allCases.enumerated() {
+      DeclSyntax(
+        """
+        /// Whether to enable the parsing of \(raw: feature.featureName).
+        public static let \(feature.token) = Self(rawValue: 1 << \(raw: i))
+        """
+      )
+    }
+  }
+}

CodeGeneration/Sources/generate-swift-syntax/templates/swiftparser/ParserTokenSpecSetFile.swift

@@ -15,6 +15,14 @@ import SwiftSyntaxBuilder
 import SyntaxSupport
 import Utils
 
+func tokenCaseMatch(_ caseName: TokenSyntax, experimentalFeature: ExperimentalFeature?) -> SwitchCaseSyntax {
+  let whereClause =
+    experimentalFeature.map {
+      "where experimentalFeatures.contains(.\($0.token))"
+    } ?? ""
+  return "case TokenSpec(.\(caseName))\(raw: whereClause): self = .\(caseName)"
+}
+
 let parserTokenSpecSetFile = SourceFileSyntax(leadingTrivia: copyrightHeader) {
   DeclSyntax("@_spi(RawSyntax) @_spi(ExperimentalLanguageFeatures) import SwiftSyntax")
 
@@ -42,16 +50,20 @@ let parserTokenSpecSetFile = SourceFileSyntax(leadingTrivia: copyrightHeader) {
               }
             }
 
-            try InitializerDeclSyntax("init?(lexeme: Lexer.Lexeme)") {
+            try InitializerDeclSyntax("init?(lexeme: Lexer.Lexeme, experimentalFeatures: Parser.ExperimentalFeatures)") {
               try SwitchExprSyntax("switch PrepareForKeywordMatch(lexeme)") {
                 for choice in choices {
                   switch choice {
                   case .keyword(let keyword):
-                    let caseName = keyword.spec.varOrCaseName
-                    SwitchCaseSyntax("case TokenSpec(.\(caseName)): self = .\(caseName)")
+                    tokenCaseMatch(
+                      keyword.spec.varOrCaseName,
+                      experimentalFeature: keyword.spec.experimentalFeature
+                    )
                   case .token(let token):
-                    let caseName = token.spec.varOrCaseName
-                    SwitchCaseSyntax("case TokenSpec(.\(caseName)): self = .\(caseName)")
+                    tokenCaseMatch(
+                      token.spec.varOrCaseName,
+                      experimentalFeature: token.spec.experimentalFeature
+                    )
                   }
                 }
                 SwitchCaseSyntax("default: return nil")

Sources/SwiftBasicFormat/BasicFormat.swift

@@ -307,7 +307,11 @@ open class BasicFormat: SyntaxRewriter {
       (.keyword(.`init`), .leftAngle),  // init<T>()
       (.keyword(.`init`), .leftParen),  // init()
       (.keyword(.self), .period),  // self.someProperty
-      (.keyword(.Self), .period),  // self.someProperty
+      (.keyword(.self), .leftParen),  // self()
+      (.keyword(.self), .leftSquare),  // self[]
+      (.keyword(.Self), .period),  // Self.someProperty
+      (.keyword(.Self), .leftParen),  // Self()
+      (.keyword(.Self), .leftSquare),  // Self[]
       (.keyword(.set), .leftParen),  // var mVar: Int { set(value) {} }
       (.keyword(.subscript), .leftParen),  // subscript(x: Int)
       (.keyword(.super), .period),  // super.someProperty

Sources/SwiftParser/Attributes.swift

@@ -64,7 +64,7 @@ extension Parser {
     case objc
     case transpose
 
-    init?(lexeme: Lexer.Lexeme) {
+    init?(lexeme: Lexer.Lexeme, experimentalFeatures: Parser.ExperimentalFeatures) {
       switch PrepareForKeywordMatch(lexeme) {
       case TokenSpec(._alignment): self = ._alignment
       case TokenSpec(._backDeploy): self = ._backDeploy
@@ -223,7 +223,7 @@ extension Parser {
       )
     }
 
-    switch DeclarationAttributeWithSpecialSyntax(lexeme: self.peek()) {
+    switch peek(isAtAnyIn: DeclarationAttributeWithSpecialSyntax.self) {
     case .available, ._spi_available:
       return parseAttribute(argumentMode: .required) { parser in
         return .availability(parser.parseAvailabilityArgumentSpecList())
@@ -1043,7 +1043,7 @@ extension Parser {
             }
           }
 
-          init?(lexeme: Lexer.Lexeme) {
+          init?(lexeme: Lexer.Lexeme, experimentalFeatures: Parser.ExperimentalFeatures) {
             switch PrepareForKeywordMatch(lexeme) {
             case TokenSpec(.private): self = .private
             case TokenSpec(.fileprivate): self = .fileprivate

Sources/SwiftParser/Availability.swift

@@ -60,7 +60,7 @@ extension Parser {
     case star
     case identifier
 
-    init?(lexeme: Lexer.Lexeme) {
+    init?(lexeme: Lexer.Lexeme, experimentalFeatures: Parser.ExperimentalFeatures) {
       switch PrepareForKeywordMatch(lexeme) {
       case TokenSpec(.message): self = .message
       case TokenSpec(.renamed): self = .renamed

Sources/SwiftParser/CMakeLists.txt

@@ -13,7 +13,6 @@ add_swift_host_library(SwiftParser
   CollectionNodes+Parsable.swift
   Declarations.swift
   Directives.swift
-  ExperimentalFeatures.swift
   Expressions.swift
   IncrementalParseTransition.swift
   Lookahead.swift
@@ -39,6 +38,7 @@ add_swift_host_library(SwiftParser
   TriviaParser.swift
   Types.swift
 
+  generated/ExperimentalFeatures.swift
   generated/IsLexerClassified.swift
   generated/LayoutNodes+Parsable.swift
   generated/Parser+TokenSpecSet.swift