Teach TokenSpecSet about experimental features

Pass down the current set of experimental features
to TokenSpecSets, and have generated TokenSpecSets
automatically account for experimental features by
returning `nil` if a given feature isn't active.

Author: hamishknight

CodeGeneration/Sources/SyntaxSupport/ExperimentalFeatures.swift

@@ -0,0 +1,33 @@
+//===----------------------------------------------------------------------===//
+//
+// This source file is part of the Swift.org open source project
+//
+// Copyright (c) 2023 Apple Inc. and the Swift project authors
+// Licensed under Apache License v2.0 with Runtime Library Exception
+//
+// See https://swift.org/LICENSE.txt for license information
+// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
+//
+//===----------------------------------------------------------------------===//
+
+import SwiftSyntax
+
+public enum ExperimentalFeature: String, CaseIterable {
+  case referenceBindings
+  case thenStatements
+
+  /// The name of the feature, which is used in the doc comment.
+  public var featureName: String {
+    switch self {
+    case .referenceBindings:
+      return "reference bindings"
+    case .thenStatements:
+      return "'then' statements"
+    }
+  }
+
+  /// The token that represents the experimental feature case name.
+  public var token: TokenSyntax {
+    .identifier(rawValue)
+  }
+}

CodeGeneration/Sources/SyntaxSupport/KeywordSpec.swift

@@ -16,14 +16,18 @@ public struct KeywordSpec {
   /// The name of the keyword.
   public let name: String
 
+  /// The experimental feature the keyword is part of, or `nil` if this isn't
+  /// for an experimental feature.
+  public let experimentalFeature: ExperimentalFeature?
+
+  /// Indicates if the token kind is switched from being an identifier to a keyword in the lexer.
+  public let isLexerClassified: Bool
+
   /// Indicates if the keyword is part of an experimental language feature.
   ///
   /// If `true`, this keyword is for an experimental language feature, and any public
   /// API generated should be marked as SPI
-  public let isExperimental: Bool
-
-  /// Indicates if the token kind is switched from being an identifier to a keyword in the lexer.
-  public let isLexerClassified: Bool
+  public var isExperimental: Bool { experimentalFeature != nil }
 
   /// The name of this keyword that's suitable to be used for variable or enum case names.
   public var varOrCaseName: TokenSyntax {
@@ -46,15 +50,15 @@ public struct KeywordSpec {
   ///
   /// - Parameters:
   ///   - name: A name of the keyword.
-  ///   - isExperimental: Indicates if the keyword is part of an experimental language feature.
+  ///   - experimentalFeature: The experimental feature the keyword is part of, or `nil` if this isn't for an experimental feature.
   ///   - isLexerClassified: Indicates if the token kind is switched from being an identifier to a keyword in the lexer.
   init(
     _ name: String,
-    isExperimental: Bool = false,
+    experimentalFeature: ExperimentalFeature? = nil,
     isLexerClassified: Bool = false
   ) {
     self.name = name
-    self.isExperimental = isExperimental
+    self.experimentalFeature = experimentalFeature
     self.isLexerClassified = isLexerClassified
   }
 }
@@ -692,11 +696,11 @@ public enum Keyword: CaseIterable {
     case .yield:
       return KeywordSpec("yield")
     case ._borrowing:
-      return KeywordSpec("_borrowing", isExperimental: true)
+      return KeywordSpec("_borrowing", experimentalFeature: .referenceBindings)
     case ._consuming:
-      return KeywordSpec("_consuming", isExperimental: true)
+      return KeywordSpec("_consuming", experimentalFeature: .referenceBindings)
     case ._mutating:
-      return KeywordSpec("_mutating", isExperimental: true)
+      return KeywordSpec("_mutating", experimentalFeature: .referenceBindings)
     }
   }
 }

CodeGeneration/Sources/SyntaxSupport/Node.swift

@@ -40,9 +40,9 @@ public class Node {
   /// The kind of node’s supertype. This kind must have `isBase == true`
   public let base: SyntaxNodeKind
 
-  /// If `true`, this is for an experimental language feature, and any public
-  /// API generated should be SPI.
-  public let isExperimental: Bool
+  /// The experimental feature the node is part of, or `nil` if this isn't
+  /// for an experimental feature.
+  public let experimentalFeature: ExperimentalFeature?
 
   /// When the node name is printed for diagnostics, this name is used.
   /// If `nil`, `nameForDiagnostics` will print the parent node’s name.
@@ -57,6 +57,10 @@ public class Node {
   /// function that should be invoked to create this node.
   public let parserFunction: TokenSyntax?
 
+  /// If `true`, this is for an experimental language feature, and any public
+  /// API generated should be SPI.
+  public var isExperimental: Bool { experimentalFeature != nil }
+
   /// A name for this node that is suitable to be used as a variables or enum
   /// case's name.
   public var varOrCaseName: TokenSyntax {
@@ -107,7 +111,7 @@ public class Node {
   init(
     kind: SyntaxNodeKind,
     base: SyntaxNodeKind,
-    isExperimental: Bool = false,
+    experimentalFeature: ExperimentalFeature? = nil,
     nameForDiagnostics: String?,
     documentation: String? = nil,
     parserFunction: TokenSyntax? = nil,
@@ -119,7 +123,7 @@ public class Node {
 
     self.kind = kind
     self.base = base
-    self.isExperimental = isExperimental
+    self.experimentalFeature = experimentalFeature
     self.nameForDiagnostics = nameForDiagnostics
     self.documentation = docCommentTrivia(from: documentation)
     self.parserFunction = parserFunction
@@ -226,7 +230,7 @@ public class Node {
   init(
     kind: SyntaxNodeKind,
     base: SyntaxNodeKind,
-    isExperimental: Bool = false,
+    experimentalFeature: ExperimentalFeature? = nil,
     nameForDiagnostics: String?,
     documentation: String? = nil,
     parserFunction: TokenSyntax? = nil,
@@ -235,7 +239,7 @@ public class Node {
     self.kind = kind
     precondition(base == .syntaxCollection)
     self.base = base
-    self.isExperimental = isExperimental
+    self.experimentalFeature = experimentalFeature
     self.nameForDiagnostics = nameForDiagnostics
     self.documentation = docCommentTrivia(from: documentation)
     self.parserFunction = parserFunction

CodeGeneration/Sources/SyntaxSupport/StmtNodes.swift

@@ -588,7 +588,7 @@ public let STMT_NODES: [Node] = [
     kind: .thenStmt,
     base: .stmt,
     // FIXME: This should be marked experimental.
-    isExperimental: false,
+    experimentalFeature: nil,
     nameForDiagnostics: "'then' statement",
     documentation: """
       A statement used to indicate the produced value from an if/switch

CodeGeneration/Sources/SyntaxSupport/TokenSpec.swift

@@ -24,11 +24,9 @@ public struct TokenSpec {
   /// The name of the token, suitable for use in variable or enum case names.
   public let varOrCaseName: TokenSyntax
 
-  /// Indicates if the token is part of an experimental language feature.
-  ///
-  /// If `true`, this keyword is for an experimental language feature, and any public
-  /// API generated should be marked as SPI
-  public let isExperimental: Bool
+  /// The experimental feature the token is part of, or `nil` if this isn't
+  /// for an experimental feature.
+  public let experimentalFeature: ExperimentalFeature?
 
   /// The name of the token that can be shown in diagnostics.
   public let nameForDiagnostics: String
@@ -39,6 +37,12 @@ public struct TokenSpec {
   /// The kind of the token.
   public let kind: Kind
 
+  /// Indicates if the token is part of an experimental language feature.
+  ///
+  /// If `true`, this keyword is for an experimental language feature, and any public
+  /// API generated should be marked as SPI
+  public var isExperimental: Bool { experimentalFeature != nil }
+
   /// The attributes that should be printed on any API for the generated keyword.
   ///
   /// This is typically used to mark APIs as SPI when the keyword is part of an experimental language feature.
@@ -51,19 +55,19 @@ public struct TokenSpec {
   ///
   /// - Parameters:
   ///   - name: A name of the token.
-  ///   - isExperimental: Indicates if the token is part of an experimental language feature.
+  ///   - experimentalFeature: The experimental feature the token is part of, or `nil` if this isn't for an experimental feature.
   ///   - nameForDiagnostics: A name of the token that can be shown in diagnostics.
   ///   - text: An actual text of the token, if available.
   ///   - kind: A kind of the token.
   fileprivate init(
     name: String,
-    isExperimental: Bool = false,
+    experimentalFeature: ExperimentalFeature? = nil,
     nameForDiagnostics: String,
     text: String? = nil,
     kind: Kind
   ) {
     self.varOrCaseName = .identifier(name)
-    self.isExperimental = isExperimental
+    self.experimentalFeature = experimentalFeature
     self.nameForDiagnostics = nameForDiagnostics
     self.text = text
     self.kind = kind

CodeGeneration/Sources/generate-swift-syntax/GenerateSwiftSyntax.swift

@@ -88,6 +88,7 @@ struct GenerateSwiftSyntax: ParsableCommand {
 
     var fileSpecs: [GeneratedFileSpec] = [
       // SwiftParser
+      GeneratedFileSpec(swiftParserGeneratedDir + ["ExperimentalFeatures.swift"], experimentalFeaturesFile),
       GeneratedFileSpec(swiftParserGeneratedDir + ["IsLexerClassified.swift"], isLexerClassifiedFile),
       GeneratedFileSpec(swiftParserGeneratedDir + ["LayoutNodes+Parsable.swift"], layoutNodesParsableFile),
       GeneratedFileSpec(swiftParserGeneratedDir + ["Parser+TokenSpecSet.swift"], parserTokenSpecSetFile),

CodeGeneration/Sources/generate-swift-syntax/templates/swiftparser/ExperimentalFeaturesFile.swift

@@ -0,0 +1,43 @@
+//===----------------------------------------------------------------------===//
+//
+// This source file is part of the Swift.org open source project
+//
+// Copyright (c) 2023 Apple Inc. and the Swift project authors
+// Licensed under Apache License v2.0 with Runtime Library Exception
+//
+// See https://swift.org/LICENSE.txt for license information
+// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
+//
+//===----------------------------------------------------------------------===//
+
+import SwiftSyntax
+import SwiftSyntaxBuilder
+import SyntaxSupport
+import Utils
+
+let experimentalFeaturesFile = SourceFileSyntax(leadingTrivia: copyrightHeader) {
+  DeclSyntax(
+    """
+    extension Parser {
+      @_spi(ExperimentalLanguageFeatures)
+      public struct ExperimentalFeatures: OptionSet {
+        public let rawValue: UInt
+        public init(rawValue: UInt) {
+          self.rawValue = rawValue
+        }
+      }
+    }
+    """
+  )
+
+  try! ExtensionDeclSyntax("extension Parser.ExperimentalFeatures") {
+    for (i, feature) in ExperimentalFeature.allCases.enumerated() {
+      DeclSyntax(
+        """
+        /// Whether to enable the parsing of \(raw: feature.featureName).
+        public static let \(feature.token) = Self(rawValue: 1 << \(raw: i))
+        """
+      )
+    }
+  }
+}

CodeGeneration/Sources/generate-swift-syntax/templates/swiftparser/ParserTokenSpecSetFile.swift

@@ -15,6 +15,14 @@ import SwiftSyntaxBuilder
 import SyntaxSupport
 import Utils
 
+func tokenCaseMatch(_ caseName: TokenSyntax, experimentalFeature: ExperimentalFeature?) -> SwitchCaseSyntax {
+  let whereClause =
+    experimentalFeature.map {
+      "where experimentalFeatures.contains(.\($0.token))"
+    } ?? ""
+  return "case TokenSpec(.\(caseName))\(raw: whereClause): self = .\(caseName)"
+}
+
 let parserTokenSpecSetFile = SourceFileSyntax(leadingTrivia: copyrightHeader) {
   DeclSyntax("@_spi(RawSyntax) @_spi(ExperimentalLanguageFeatures) import SwiftSyntax")
 
@@ -42,16 +50,20 @@ let parserTokenSpecSetFile = SourceFileSyntax(leadingTrivia: copyrightHeader) {
               }
             }
 
-            try InitializerDeclSyntax("init?(lexeme: Lexer.Lexeme)") {
+            try InitializerDeclSyntax("init?(lexeme: Lexer.Lexeme, experimentalFeatures: Parser.ExperimentalFeatures)") {
               try SwitchExprSyntax("switch PrepareForKeywordMatch(lexeme)") {
                 for choice in choices {
                   switch choice {
                   case .keyword(let keyword):
-                    let caseName = keyword.spec.varOrCaseName
-                    SwitchCaseSyntax("case TokenSpec(.\(caseName)): self = .\(caseName)")
+                    tokenCaseMatch(
+                      keyword.spec.varOrCaseName,
+                      experimentalFeature: keyword.spec.experimentalFeature
+                    )
                   case .token(let token):
-                    let caseName = token.spec.varOrCaseName
-                    SwitchCaseSyntax("case TokenSpec(.\(caseName)): self = .\(caseName)")
+                    tokenCaseMatch(
+                      token.spec.varOrCaseName,
+                      experimentalFeature: token.spec.experimentalFeature
+                    )
                   }
                 }
                 SwitchCaseSyntax("default: return nil")

Sources/SwiftParser/Attributes.swift

@@ -64,7 +64,7 @@ extension Parser {
     case objc
     case transpose
 
-    init?(lexeme: Lexer.Lexeme) {
+    init?(lexeme: Lexer.Lexeme, experimentalFeatures: Parser.ExperimentalFeatures) {
       switch PrepareForKeywordMatch(lexeme) {
       case TokenSpec(._alignment): self = ._alignment
       case TokenSpec(._backDeploy): self = ._backDeploy
@@ -1043,7 +1043,7 @@ extension Parser {
             }
           }
 
-          init?(lexeme: Lexer.Lexeme) {
+          init?(lexeme: Lexer.Lexeme, experimentalFeatures: Parser.ExperimentalFeatures) {
             switch PrepareForKeywordMatch(lexeme) {
             case TokenSpec(.private): self = .private
             case TokenSpec(.fileprivate): self = .fileprivate

Sources/SwiftParser/Availability.swift

@@ -60,7 +60,7 @@ extension Parser {
     case star
     case identifier
 
-    init?(lexeme: Lexer.Lexeme) {
+    init?(lexeme: Lexer.Lexeme, experimentalFeatures: Parser.ExperimentalFeatures) {
       switch PrepareForKeywordMatch(lexeme) {
       case TokenSpec(.message): self = .message
       case TokenSpec(.renamed): self = .renamed

Sources/SwiftParser/Declarations.swift

@@ -521,7 +521,7 @@ extension Parser {
           case postfixOperator
           case prefixOperator
 
-          init?(lexeme: Lexer.Lexeme) {
+          init?(lexeme: Lexer.Lexeme, experimentalFeatures: Parser.ExperimentalFeatures) {
             switch (lexeme.rawTokenKind, lexeme.tokenText) {
             case (.colon, _): self = .colon
             case (.binaryOperator, "=="): self = .binaryOperator
@@ -1719,7 +1719,7 @@ extension Parser {
       case higherThan
       case lowerThan
 
-      init?(lexeme: Lexer.Lexeme) {
+      init?(lexeme: Lexer.Lexeme, experimentalFeatures: Parser.ExperimentalFeatures) {
         switch PrepareForKeywordMatch(lexeme) {
         case TokenSpec(.associativity): self = .associativity
         case TokenSpec(.assignment): self = .assignment

Sources/SwiftParser/Directives.swift

@@ -26,7 +26,7 @@ extension Parser {
       }
     }
 
-    init?(lexeme: Lexer.Lexeme) {
+    init?(lexeme: Lexer.Lexeme, experimentalFeatures: Parser.ExperimentalFeatures) {
       switch PrepareForKeywordMatch(lexeme) {
       case TokenSpec(.poundElseif): self = .poundElseif
       case TokenSpec(.poundElse): self = .poundElse

Sources/SwiftParser/Expressions.swift

@@ -244,7 +244,7 @@ extension Parser {
       case arrow
       case `throws`
 
-      init?(lexeme: Lexer.Lexeme) {
+      init?(lexeme: Lexer.Lexeme, experimentalFeatures: Parser.ExperimentalFeatures) {
         switch PrepareForKeywordMatch(lexeme) {
         case TokenSpec(.binaryOperator): self = .binaryOperator
         case TokenSpec(.infixQuestionMark): self = .infixQuestionMark

Sources/SwiftParser/Lookahead.swift

@@ -333,7 +333,7 @@ extension Parser.Lookahead {
     case poundElse
     case poundElseif
 
-    init?(lexeme: Lexer.Lexeme) {
+    init?(lexeme: Lexer.Lexeme, experimentalFeatures: Parser.ExperimentalFeatures) {
       switch lexeme.rawTokenKind {
       case .leftParen: self = .leftParen
       case .leftBrace: self = .leftBrace

Sources/SwiftParser/Modifiers.swift

@@ -169,7 +169,7 @@ extension Parser {
         }
       }
 
-      init?(lexeme: Lexer.Lexeme) {
+      init?(lexeme: Lexer.Lexeme, experimentalFeatures: Parser.ExperimentalFeatures) {
         switch PrepareForKeywordMatch(lexeme) {
         case TokenSpec(.private): self = .private
         case TokenSpec(.fileprivate): self = .fileprivate

Sources/SwiftParser/Names.swift

@@ -304,7 +304,9 @@ extension Lexer.Lexeme {
   }
 
   func isContextualPunctuator(_ name: SyntaxText) -> Bool {
-    return Operator(lexeme: self) != nil && self.tokenText == name
+    // Currently we can ignore experimental features since a new kind of
+    // non-prefix/infix/postfix operator seems unlikely.
+    return Operator(lexeme: self, experimentalFeatures: []) != nil && self.tokenText == name
   }
 
   var isLexerClassifiedKeyword: Bool {