Removed TODOs in testRecovery102/103/104

Author: CippoX

BUILD.bazel

@@ -3,7 +3,7 @@ load("//utils/bazel:swift_syntax_library.bzl", "swift_syntax_library")
 package(default_visibility = ["//visibility:public"])
 
 swift_syntax_library(
-    name = "IDEUtils",
+    name = "SwiftIDEUtils",
     deps = [
         ":SwiftSyntax",
     ],
@@ -95,14 +95,6 @@ swift_syntax_library(
     ],
 )
 
-swift_syntax_library(
-    name = "SwiftSyntaxParser",
-    deps = [
-        ":SwiftParser",
-        ":SwiftSyntax",
-    ],
-)
-
 swift_syntax_library(
     name = "SwiftRefactor",
     deps = [

CONTRIBUTING.md

@@ -14,6 +14,7 @@ Alternatively you can also build it from the command line using `build-script.py
      - (enclosing directory)
        - swift-argument-parser
        - swift-syntax
+     ```
 2. Execute the following command
     ```bash
     swift-syntax/build-script.py build --toolchain /Library/Developer/Toolchains/swift-DEVELOPMENT-SNAPSHOT-<recent date>.xctoolchain/usr
@@ -43,11 +44,7 @@ If you can’t find it in your Schemes, you need to manually add it using Produc
 
 ### XCTests
 
-The `SwiftSyntaxParser` module (the legacy parser) of this repository depends on the C++ parser library (`_InternalSwiftSyntaxParser.dylib`) to parse source code.
-The syntax node definitions of that parser library need to match those used by your SwiftSyntax checkout.
-Most of the time, the parser library included in the latest Swift Development Snapshot will fulfill this requirement. 
-
-To run the tests in Xcode, select the latest Swift Development Snapshot in Xcode -> Toolchains, select the SwiftSyntax-Package scheme and hit Product -> Test.
+To run the tests in Xcode, select the SwiftSyntax-Package scheme and hit Product -> Test.
 
 You can also run the tests from the command line using
 ```bash
@@ -66,15 +63,15 @@ Tip: Running SwiftSyntax’s self-parse tests takes the majority of testing time
 2. Select the Arguments tab in the Run section
 3. Add a `SKIP_LONG_TESTS` environment variable with value `1`
 
-### `lit`-based tests
+### `lit`-based Tests
 
-A few tests of the `SwiftSyntaxParser` module (the legacy parser), which test the interaction between SwiftSyntax and the C++ parser library (`_InternalSwiftSyntaxParser.dylib`) are based LLVM’s `lit` and `FileCheck` tools.
+A few tests are based LLVM’s `lit` and `FileCheck` tools.
 To run these, build `FileCheck`, e.g. by building the Swift compiler and run the tests using the following command:
 ```bash
 ./build-script.py test --toolchain /Library/Developer/Toolchains/swift-DEVELOPMENT-SNAPSHOT-<recent date>.xctoolchain/usr --skip-lit-tests --filecheck-exec /path/to/build/Release+Asserts/llvm-macosx-x86_64/bin/FileCheck
 ```
 
-## Generating source code
+## Generating Source Code
 
 If you want to modify the code-generated files, perform the following steps:
 
@@ -90,3 +87,7 @@ If you want to modify the code-generated files, perform the following steps:
     swift-syntax/build-script.py generate-source-code --toolchain /Library/Developer/Toolchains/swift-DEVELOPMENT-SNAPSHOT-<recent date>.xctoolchain/usr
     ```
 3. The new source-generated file will be written into your `Sources` directory.
+
+## Swift Version
+
+We require that SwiftSyntax builds with the latest released compiler and the previous major version (e.g. with Swift 5.8 and Swift 5.7).

CodeGeneration/Package.swift

@@ -11,8 +11,8 @@ let package = Package(
     .executable(name: "generate-swiftsyntax", targets: ["generate-swiftsyntax"])
   ],
   dependencies: [
-    .package(url: "https://github.com/apple/swift-syntax.git", revision: "39b3336c3f3bfcd4ddbcbf6a111d8814ffe542f3"),
-    .package(url: "https://github.com/apple/swift-argument-parser.git", .upToNextMinor(from: "1.2.2")),
+    .package(url: "https://github.com/apple/swift-syntax.git", revision: "c7087adb193b26f02b59d21cd06d17ec202d1bf5"),
+    .package(url: "https://github.com/apple/swift-argument-parser.git", from: "1.2.2"),
   ],
   targets: [
     .executableTarget(

CodeGeneration/Sources/SyntaxSupport/AttributeNodes.swift

@@ -388,7 +388,7 @@ public let ATTRIBUTE_NODES: [Node] = [
     children: [
       Child(
         name: "Parameter",
-        kind: .token(choices: [.token(tokenKind: "IdentifierToken"), .token(tokenKind: "IntegerLiteralToken"), .token(tokenKind: "KeywordToken")])
+        kind: .token(choices: [.token(tokenKind: "IdentifierToken"), .token(tokenKind: "IntegerLiteralToken"), .keyword(text: "self")])
       ),
       Child(
         name: "TrailingComma",
@@ -518,7 +518,7 @@ public let ATTRIBUTE_NODES: [Node] = [
         kind: .nodeChoices(choices: [
           Child(
             name: "Token",
-            kind: .token(choices: [.token(tokenKind: "IdentifierToken"), .token(tokenKind: "KeywordToken")])
+            kind: .token(choices: [.token(tokenKind: "IdentifierToken"), .keyword(text: "private"), .keyword(text: "fileprivate"), .keyword(text: "internal"), .keyword(text: "public"), .keyword(text: "open")])
           ),  // Keywords can be: public, internal, private, fileprivate, open
           Child(
             name: "String",

CodeGeneration/Sources/SyntaxSupport/CommonNodes.swift

@@ -16,8 +16,7 @@ public let COMMON_NODES: [Node] = [
     name: "CodeBlockItemList",
     nameForDiagnostics: nil,
     kind: "SyntaxCollection",
-    element: "CodeBlockItem",
-    elementsSeparatedByNewline: true
+    element: "CodeBlockItem"
   ),
 
   // code-block-item = (decl | stmt | expr) ';'?
@@ -144,6 +143,7 @@ public let COMMON_NODES: [Node] = [
   Node(
     name: "MissingDecl",
     nameForDiagnostics: "declaration",
+    description: "In case the source code is missing a declaration, this node stands in place of the missing declaration.",
     kind: "Decl",
     traits: [
       "Attributed"
@@ -152,44 +152,108 @@ public let COMMON_NODES: [Node] = [
       Child(
         name: "Attributes",
         kind: .collection(kind: "AttributeList", collectionElementName: "Attribute"),
+        description: "If there were standalone attributes without a declaration to attach them to, the `MissingDeclSyntax` will contain these.",
         isOptional: true
       ),
       Child(
         name: "Modifiers",
         kind: .collection(kind: "ModifierList", collectionElementName: "Modifier"),
+        description: "If there were standalone modifiers without a declaration to attach them to, the `MissingDeclSyntax` will contain these.",
         isOptional: true
       ),
+      Child(
+        name: "Placeholder",
+        kind: .token(choices: [.token(tokenKind: "IdentifierToken")], requiresLeadingSpace: false, requiresTrailingSpace: false),
+        description: """
+          A placeholder, i.e. `<#decl#>` that can be inserted into the source code to represent the missing declaration.
+          This token should always have `presence = .missing`.
+          """
+      ),
     ]
   ),
 
   Node(
     name: "MissingExpr",
     nameForDiagnostics: "expression",
-    kind: "Expr"
+    description: "In case the source code is missing a expression, this node stands in place of the missing expression.",
+    kind: "Expr",
+    children: [
+      Child(
+        name: "Placeholder",
+        kind: .token(choices: [.token(tokenKind: "IdentifierToken")], requiresLeadingSpace: false, requiresTrailingSpace: false),
+        description: """
+          A placeholder, i.e. `<#expression#>` that can be inserted into the source code to represent the missing expression.
+          This token should always have `presence = .missing`.
+          """
+      )
+    ]
   ),
 
   Node(
     name: "MissingPattern",
     nameForDiagnostics: "pattern",
-    kind: "Pattern"
+    description: "In case the source code is missing a pattern, this node stands in place of the missing pattern.",
+    kind: "Pattern",
+    children: [
+      Child(
+        name: "Placeholder",
+        kind: .token(choices: [.token(tokenKind: "IdentifierToken")], requiresLeadingSpace: false, requiresTrailingSpace: false),
+        description: """
+          A placeholder, i.e. `<#pattern#>` that can be inserted into the source code to represent the missing pattern.
+          This token should always have `presence = .missing`.
+          """
+      )
+    ]
   ),
 
   Node(
     name: "MissingStmt",
     nameForDiagnostics: "statement",
-    kind: "Stmt"
+    description: "In case the source code is missing a statement, this node stands in place of the missing statement.",
+    kind: "Stmt",
+    children: [
+      Child(
+        name: "Placeholder",
+        kind: .token(choices: [.token(tokenKind: "IdentifierToken")], requiresLeadingSpace: false, requiresTrailingSpace: false),
+        description: """
+          A placeholder, i.e. `<#statement#>` that can be inserted into the source code to represent the missing pattern.
+          This token should always have `presence = .missing`.
+          """
+      )
+    ]
   ),
 
   Node(
     name: "Missing",
     nameForDiagnostics: nil,
-    kind: "Syntax"
+    description: "In case the source code is missing a syntax node, this node stands in place of the missing node.",
+    kind: "Syntax",
+    children: [
+      Child(
+        name: "Placeholder",
+        kind: .token(choices: [.token(tokenKind: "IdentifierToken")], requiresLeadingSpace: false, requiresTrailingSpace: false),
+        description: """
+          A placeholder, i.e. `<#syntax#>` that can be inserted into the source code to represent the missing pattern.
+          This token should always have `presence = .missing`
+          """
+      )
+    ]
   ),
 
   Node(
     name: "MissingType",
     nameForDiagnostics: "type",
-    kind: "Type"
+    description: "In case the source code is missing a type, this node stands in place of the missing type.",
+    kind: "Type",
+    children: [
+      Child(
+        name: "Placeholder",
+        kind: .token(choices: [.token(tokenKind: "IdentifierToken")], requiresLeadingSpace: false, requiresTrailingSpace: false),
+        description: """
+          A placeholder, i.e. `<#type#>` that can be inserted into the source code to represent the missing type. This token should always have `presence = .missing`.
+          """
+      )
+    ]
   ),
 
   Node(

CodeGeneration/Sources/SyntaxSupport/DeclNodes.swift

@@ -111,8 +111,7 @@ public let DECL_NODES: [Node] = [
     name: "AccessorList",
     nameForDiagnostics: nil,
     kind: "SyntaxCollection",
-    element: "AccessorDecl",
-    elementsSeparatedByNewline: true
+    element: "AccessorDecl"
   ),
 
   // (value)
@@ -1029,14 +1028,6 @@ public let DECL_NODES: [Node] = [
       "WithTrailingComma"
     ],
     children: [
-      /// Indicates whether the 'without' operator was applied to the type to
-      /// indicate the suppression of implicit conformance to this type.
-      /// This child stores the token representing the 'without' operator.
-      Child(
-        name: "HasWithout",
-        kind: .token(choices: [.token(tokenKind: "PrefixOperatorToken")]),
-        isOptional: true
-      ),
       Child(
         name: "TypeName",
         kind: .node(kind: "Type")
@@ -1296,8 +1287,7 @@ public let DECL_NODES: [Node] = [
     name: "MemberDeclList",
     nameForDiagnostics: nil,
     kind: "SyntaxCollection",
-    element: "MemberDeclListItem",
-    elementsSeparatedByNewline: true
+    element: "MemberDeclListItem"
   ),
 
   // declaration-modifier -> access-level-modifier

CodeGeneration/Sources/SyntaxSupport/ExprNodes.swift

@@ -781,7 +781,7 @@ public let EXPR_NODES: [Node] = [
     children: [
       Child(
         name: "Identifier",
-        kind: .token(choices: [.token(tokenKind: "IdentifierToken"), .token(tokenKind: "KeywordToken"), .token(tokenKind: "DollarIdentifierToken"), .token(tokenKind: "BinaryOperatorToken")])
+        kind: .token(choices: [.token(tokenKind: "IdentifierToken"), .keyword(text: "self"), .keyword(text: "Self"), .keyword(text: "init"), .token(tokenKind: "DollarIdentifierToken"), .token(tokenKind: "BinaryOperatorToken")])
       ),
       Child(
         name: "DeclNameArguments",
@@ -1005,7 +1005,7 @@ public let EXPR_NODES: [Node] = [
     children: [
       Child(
         name: "Identifier",
-        kind: .token(choices: [.token(tokenKind: "IdentifierToken"), .token(tokenKind: "KeywordToken"), .token(tokenKind: "DollarIdentifierToken"), .token(tokenKind: "BinaryOperatorToken"), .token(tokenKind: "IntegerLiteralToken")])
+        kind: .token(choices: [.token(tokenKind: "IdentifierToken"), .keyword(text: "self"), .keyword(text: "Self"), .keyword(text: "init"), .token(tokenKind: "DollarIdentifierToken"), .token(tokenKind: "BinaryOperatorToken"), .token(tokenKind: "IntegerLiteralToken")])
       ),
       Child(
         name: "DeclNameArguments",
@@ -1481,8 +1481,7 @@ public let EXPR_NODES: [Node] = [
     kind: "SyntaxCollection",
     element: "Syntax",
     elementName: "SwitchCase",
-    elementChoices: ["SwitchCase", "IfConfigDecl"],
-    elementsSeparatedByNewline: true
+    elementChoices: ["SwitchCase", "IfConfigDecl"]
   ),
 
   // switch-case -> unknown-attr? switch-case-label stmt-list

CodeGeneration/Sources/SyntaxSupport/Node.swift

@@ -30,7 +30,6 @@ public class Node {
   public let collectionElementName: String?
   public let collectionElementChoices: [String]?
   public let omitWhenEmpty: Bool
-  public let elementsSeparatedByNewline: Bool
   public let collectionElement: String
 
   /// Returns `true` if this node declares one of the base syntax kinds.
@@ -98,8 +97,7 @@ public class Node {
     element: String = "",
     elementName: String? = nil,
     elementChoices: [String]? = nil,
-    omitWhenEmpty: Bool = false,
-    elementsSeparatedByNewline: Bool = false
+    omitWhenEmpty: Bool = false
   ) {
     self.syntaxKind = name
     self.swiftSyntaxKind = lowercaseFirstWord(name: name)
@@ -159,7 +157,6 @@ public class Node {
     // from its supertype, use that.
     self.collectionElementName = elementName ?? self.collectionElement
     self.collectionElementChoices = elementChoices ?? []
-    self.elementsSeparatedByNewline = elementsSeparatedByNewline
 
     // For SyntaxCollections make sure that the elementName is set.
     precondition(!isSyntaxCollection || elementName != nil || element != "")

CodeGeneration/Sources/SyntaxSupport/PatternNodes.swift

@@ -31,7 +31,7 @@ public let PATTERN_NODES: [Node] = [
     children: [
       Child(
         name: "Identifier",
-        kind: .token(choices: [.token(tokenKind: "IdentifierToken"), .token(tokenKind: "KeywordToken")])
+        kind: .token(choices: [.token(tokenKind: "IdentifierToken"), .keyword(text: "self"), .keyword(text: "init")])
       )
     ]
   ),

CodeGeneration/Sources/SyntaxSupport/Traits.swift

@@ -48,9 +48,9 @@ public let TRAITS: [Trait] = [
     traitName: "EffectSpecifiers",
     children: [
       Child(name: "UnexpectedBeforeAsyncSpecifier", kind: .node(kind: "UnexpectedNodes"), isOptional: true),
-      Child(name: "AsyncSpecifier", kind: .token(choices: [.token(tokenKind: "KeywordToken")]), isOptional: true),
+      Child(name: "AsyncSpecifier", kind: .token(choices: [.keyword(text: "async"), .keyword(text: "reasync")]), isOptional: true),
       Child(name: "UnexpectedBetweenAsyncSpecifierAndThrowsSpecifier", kind: .node(kind: "UnexpectedNodes"), isOptional: true),
-      Child(name: "ThrowsSpecifier", kind: .token(choices: [.token(tokenKind: "KeywordToken")]), isOptional: true),
+      Child(name: "ThrowsSpecifier", kind: .token(choices: [.keyword(text: "throws"), .keyword(text: "rethrows")]), isOptional: true),
       Child(name: "UnexpectedAfterThrowsSpecifier", kind: .node(kind: "UnexpectedNodes"), isOptional: true),
     ]
   ),