Merge pull request #1706 from ahoppen/ahoppen/flip-token-presence

Add a new test mutation strategy to flip the presence of token in the `assertParse` tests (fixes 5 parser bugs)

Author: ahoppen

CodeGeneration/Sources/SyntaxSupport/DeclNodes.swift

@@ -549,11 +549,33 @@ public let DECL_NODES: [Node] = [
     kind: .editorPlaceholderDecl,
     base: .decl,
     nameForDiagnostics: "editor placeholder",
+    documentation: """
+      An editor placeholder, e.g. `<#declaration#>` that is used in a position that expects a declaration.
+      """,
+    traits: [
+      "WithAttributes",
+      "WithModifiers",
+    ],
     children: [
       Child(
-        name: "Identifier",
-        kind: .token(choices: [.token(tokenKind: "IdentifierToken")])
-      )
+        name: "Attributes",
+        kind: .collection(kind: .attributeList, collectionElementName: "Attribute"),
+        documentation: "If there were attributes before the editor placeholder, the ``EditorPlaceholderDecl`` will contain these.",
+        isOptional: true
+      ),
+      Child(
+        name: "Modifiers",
+        kind: .collection(kind: .modifierList, collectionElementName: "Modifier"),
+        documentation: "If there were modifiers before the editor placeholder, the `EditorPlaceholderDecl` will contain these.",
+        isOptional: true
+      ),
+      Child(
+        name: "Placeholder",
+        kind: .token(choices: [.token(tokenKind: "IdentifierToken")]),
+        documentation: """
+          The actual editor placeholder that starts with `<#` and ends with `#>`.
+          """
+      ),
     ]
   ),
 

Sources/SwiftParser/Declarations.swift

@@ -286,7 +286,9 @@ extension Parser {
           let placeholder = self.consumeAnyToken()
           return RawDeclSyntax(
             RawEditorPlaceholderDeclSyntax(
-              identifier: placeholder,
+              attributes: attrs.attributes,
+              modifiers: attrs.modifiers,
+              placeholder: placeholder,
               arena: self.arena
             )
           )
@@ -474,8 +476,12 @@ extension Parser {
       )
     }
 
-    precondition(self.currentToken.starts(with: "<"))
-    let langle = self.consumePrefix("<", as: .leftAngle)
+    let langle: RawTokenSyntax
+    if self.currentToken.starts(with: "<") {
+      langle = self.consumePrefix("<", as: .leftAngle)
+    } else {
+      langle = missingToken(.leftAngle)
+    }
     var elements = [RawGenericParameterSyntax]()
     do {
       var keepGoing: RawTokenSyntax? = nil

Sources/SwiftParser/Nominals.swift

@@ -333,7 +333,7 @@ extension Parser {
   }
 
   mutating func parsePrimaryAssociatedTypes() -> RawPrimaryAssociatedTypeClauseSyntax {
-    let langle = self.consumeAnyToken(remapping: .leftAngle)
+    let langle = self.consumePrefix("<", as: .leftAngle)
     var associatedTypes = [RawPrimaryAssociatedTypeSyntax]()
     do {
       var keepGoing: RawTokenSyntax? = nil

Sources/SwiftParser/Types.swift

@@ -546,7 +546,7 @@ extension Parser {
               inOut: nil,
               name: nil,
               secondName: nil,
-              unexpectedBeforeColon,
+              RawUnexpectedNodesSyntax(combining: misplacedSpecifiers, unexpectedBeforeColon, arena: self.arena),
               colon: nil,
               type: RawTypeSyntax(RawSimpleTypeIdentifierSyntax(name: first, genericArgumentClause: nil, arena: self.arena)),
               ellipsis: nil,

Sources/SwiftParserDiagnostics/MissingTokenError.swift

@@ -17,6 +17,7 @@ import SwiftDiagnostics
 extension ParseDiagnosticsGenerator {
   func handleMissingToken(_ missingToken: TokenSyntax) {
     guard let invalidToken = missingToken.previousToken(viewMode: .all),
+      invalidToken.presence == .present,
       let invalidTokenContainer = invalidToken.parent?.as(UnexpectedNodesSyntax.self),
       invalidTokenContainer.count == 1
     else {

Sources/SwiftParserDiagnostics/ParseDiagnosticsGenerator.swift

@@ -15,6 +15,26 @@ import SwiftDiagnostics
 @_spi(RawSyntax) import SwiftSyntax
 
 fileprivate func getTokens(between first: TokenSyntax, and second: TokenSyntax) -> [TokenSyntax] {
+  var first = first
+  if first.presence == .missing {
+    let nextPresentToken = first.nextToken(viewMode: .sourceAccurate)
+    guard let nextPresentToken else {
+      return []
+    }
+    first = nextPresentToken
+  }
+  precondition(first.presence == .present)
+
+  var second = second
+  if second.presence == .missing {
+    let previousPresentToken = second.previousToken(viewMode: .sourceAccurate)
+    guard let previousPresentToken else {
+      return []
+    }
+    second = previousPresentToken
+  }
+  precondition(second.presence == .present)
+
   var tokens: [TokenSyntax] = []
   var currentToken = first
 
@@ -149,7 +169,7 @@ public class ParseDiagnosticsGenerator: SyntaxAnyVisitor {
     removeRedundantFixIt: (_ misplacedTokens: [TokenSyntax]) -> FixItMessage? = { _ in nil }
   ) {
     guard let incorrectContainer = unexpected,
-      let misplacedTokens = incorrectContainer.onlyTokens(satisfying: unexpectedTokenCondition)
+      let misplacedTokens = incorrectContainer.onlyPresentTokens(satisfying: unexpectedTokenCondition)
     else {
       // If there are no unexpected nodes or the unexpected contain multiple tokens, don't emit a diagnostic.
       return
@@ -197,7 +217,7 @@ public class ParseDiagnosticsGenerator: SyntaxAnyVisitor {
     message: (TokenSyntax) -> some DiagnosticMessage
   ) {
     guard let unexpected = unexpected,
-      let misplacedToken = unexpected.onlyToken(where: predicate)
+      let misplacedToken = unexpected.onlyPresentToken(where: predicate)
     else {
       // If there is no unexpected node or the unexpected doesn't have the
       // expected token, don't emit a diagnostic.
@@ -279,7 +299,7 @@ public class ParseDiagnosticsGenerator: SyntaxAnyVisitor {
       }
       if specifier.presence == .present {
         for case .some(let unexpected) in unexpectedNodes {
-          for duplicateSpecifier in unexpected.tokens(satisfying: isOfSameKind) {
+          for duplicateSpecifier in unexpected.presentTokens(satisfying: isOfSameKind) {
             addDiagnostic(
               duplicateSpecifier,
               DuplicateEffectSpecifiers(correctSpecifier: specifier, unexpectedSpecifier: duplicateSpecifier),
@@ -315,29 +335,36 @@ public class ParseDiagnosticsGenerator: SyntaxAnyVisitor {
       suppressRemainingDiagnostics = true
       return .skipChildren
     }
-    if let tryKeyword = node.onlyToken(where: { $0.tokenKind == .keyword(.try) }),
+    if let tryKeyword = node.onlyPresentToken(where: { $0.tokenKind == .keyword(.try) }),
       let nextToken = tryKeyword.nextToken(viewMode: .sourceAccurate),
       nextToken.tokenKind.isLexerClassifiedKeyword,
       !(node.parent?.is(TypeEffectSpecifiersSyntax.self) ?? false)
     {
       addDiagnostic(node, TryCannotBeUsed(nextToken: nextToken))
-    } else if let semicolons = node.onlyTokens(satisfying: { $0.tokenKind == .semicolon }) {
+    } else if let semicolons = node.onlyPresentTokens(satisfying: { $0.tokenKind == .semicolon }) {
       addDiagnostic(
         node,
         .unexpectedSemicolon,
         fixIts: [
           FixIt(message: RemoveNodesFixIt(semicolons), changes: semicolons.map { FixIt.MultiNodeChange.makeMissing($0) })
         ]
       )
-    } else if node.first?.as(TokenSyntax.self)?.tokenKind.isIdentifier == true,
+    } else if let firstToken = node.first?.as(TokenSyntax.self),
+      firstToken.tokenKind.isIdentifier == true,
+      firstToken.presence == .present,
       let previousToken = node.previousToken(viewMode: .sourceAccurate),
       previousToken.tokenKind.isIdentifier,
       previousToken.parent?.is(DeclSyntax.self) == true || previousToken.parent?.is(IdentifierPatternSyntax.self) == true
     {
       // If multiple identifiers are used for a declaration name, offer to join them together.
       let tokens =
         node
-        .prefix(while: { $0.as(TokenSyntax.self)?.tokenKind.isIdentifier == true })
+        .prefix(while: {
+          guard let token = $0.as(TokenSyntax.self) else {
+            return false
+          }
+          return token.tokenKind.isIdentifier == true && token.presence == .present
+        })
         .map({ $0.as(TokenSyntax.self)! })
       let joined = previousToken.text + tokens.map(\.text).joined()
       var fixIts: [FixIt] = [
@@ -494,7 +521,7 @@ public class ParseDiagnosticsGenerator: SyntaxAnyVisitor {
     }
 
     if let unexpectedAfterRightParen = node.unexpectedAfterRightParen,
-      let (_, falseKeyword) = unexpectedAfterRightParen.twoTokens(
+      let (_, falseKeyword) = unexpectedAfterRightParen.twoPresentTokens(
         firstSatisfying: { $0.tokenKind == .binaryOperator("==") },
         secondSatisfying: { $0.tokenKind == .keyword(.false) }
       )
@@ -531,7 +558,7 @@ public class ParseDiagnosticsGenerator: SyntaxAnyVisitor {
       return .skipChildren
     }
     if let unexpected = node.unexpectedBetweenPlatformAndVersion,
-      unexpected.onlyToken(where: { $0.tokenKind == .binaryOperator(">=") }) != nil
+      unexpected.onlyPresentToken(where: { $0.tokenKind == .binaryOperator(">=") }) != nil
     {
       addDiagnostic(
         unexpected,
@@ -678,7 +705,7 @@ public class ParseDiagnosticsGenerator: SyntaxAnyVisitor {
     }
 
     if let unexpected = node.unexpectedBetweenBodyAndTrailingComma,
-      let token = unexpected.tokens(satisfying: { $0.tokenKind == .binaryOperator("&&") }).first,
+      let token = unexpected.presentTokens(satisfying: { $0.tokenKind == .binaryOperator("&&") }).first,
       let trailingComma = node.trailingComma,
       trailingComma.presence == .missing,
       let previous = node.unexpectedBetweenBodyAndTrailingComma?.previousToken(viewMode: .sourceAccurate)
@@ -709,7 +736,7 @@ public class ParseDiagnosticsGenerator: SyntaxAnyVisitor {
       return .skipChildren
     }
     if let unexpected = node.unexpectedBetweenDeinitKeywordAndBody,
-      let name = unexpected.filter({ $0.as(TokenSyntax.self)?.tokenKind.isIdentifier == true }).only?.as(TokenSyntax.self)
+      let name = unexpected.presentTokens(satisfying: { $0.tokenKind.isIdentifier == true }).only?.as(TokenSyntax.self)
     {
       addDiagnostic(
         name,
@@ -741,7 +768,7 @@ public class ParseDiagnosticsGenerator: SyntaxAnyVisitor {
       return .skipChildren
     }
     if node.floatingDigits.presence == .missing,
-      let (period, integerLiteral) = node.unexpectedAfterFloatingDigits?.twoTokens(
+      let (period, integerLiteral) = node.unexpectedAfterFloatingDigits?.twoPresentTokens(
         firstSatisfying: { $0.tokenKind == .period },
         secondSatisfying: { $0.tokenKind.isIntegerLiteral }
       )
@@ -772,7 +799,7 @@ public class ParseDiagnosticsGenerator: SyntaxAnyVisitor {
     // Detect C-style for loops based on two semicolons which could not be parsed between the 'for' keyword and the '{'
     // This is mostly a proof-of-concept implementation to produce more complex diagnostics.
     if let unexpectedCondition = node.body.unexpectedBeforeLeftBrace,
-      unexpectedCondition.tokens(withKind: .semicolon).count == 2
+      unexpectedCondition.presentTokens(withKind: .semicolon).count == 2
     {
       // FIXME: This is aweful. We should have a way to either get all children between two cursors in a syntax node or highlight a range from one node to another.
       addDiagnostic(
@@ -855,7 +882,7 @@ public class ParseDiagnosticsGenerator: SyntaxAnyVisitor {
         message: { _ in .typeParameterPackEllipsis }
       )
     } else if let unexpected = node.unexpectedBetweenNameAndColon,
-      let unexpectedEllipsis = unexpected.onlyToken(where: { $0.tokenKind == .ellipsis }),
+      let unexpectedEllipsis = unexpected.onlyPresentToken(where: { $0.tokenKind == .ellipsis }),
       let each = node.each
     {
       addDiagnostic(
@@ -1105,7 +1132,7 @@ public class ParseDiagnosticsGenerator: SyntaxAnyVisitor {
     if shouldSkip(node) {
       return .skipChildren
     }
-    if let token = node.unexpectedBetweenModuleLabelAndColon?.onlyToken(where: { $0.tokenKind.isIdentifier }),
+    if let token = node.unexpectedBetweenModuleLabelAndColon?.onlyPresentToken(where: { $0.tokenKind.isIdentifier }),
       node.moduleLabel.presence == .missing
     {
       addDiagnostic(
@@ -1172,9 +1199,9 @@ public class ParseDiagnosticsGenerator: SyntaxAnyVisitor {
         return
       }
       let message: DiagnosticMessage?
-      if let identifier = unexpected.onlyToken(where: { $0.tokenKind.isIdentifier }) {
+      if let identifier = unexpected.onlyPresentToken(where: { $0.tokenKind.isIdentifier }) {
         message = IdentifierNotAllowedInOperatorName(identifier: identifier)
-      } else if let tokens = unexpected.onlyTokens(satisfying: { _ in true }) {
+      } else if let tokens = unexpected.onlyPresentTokens(satisfying: { _ in true }) {
         message = TokensNotAllowedInOperatorName(tokens: tokens)
       } else {
         message = nil
@@ -1265,7 +1292,7 @@ public class ParseDiagnosticsGenerator: SyntaxAnyVisitor {
       return .skipChildren
     }
     // recover from Objective-C style literals
-    if let atSign = node.unexpectedBetweenOpenDelimiterAndOpenQuote?.onlyToken(where: { $0.tokenKind == .atSign }) {
+    if let atSign = node.unexpectedBetweenOpenDelimiterAndOpenQuote?.onlyPresentToken(where: { $0.tokenKind == .atSign }) {
       addDiagnostic(
         node,
         .stringLiteralAtSign,
@@ -1275,7 +1302,7 @@ public class ParseDiagnosticsGenerator: SyntaxAnyVisitor {
         handledNodes: [atSign.id]
       )
     }
-    if let singleQuote = node.unexpectedBetweenOpenDelimiterAndOpenQuote?.onlyToken(where: { $0.tokenKind == .singleQuote }) {
+    if let singleQuote = node.unexpectedBetweenOpenDelimiterAndOpenQuote?.onlyPresentToken(where: { $0.tokenKind == .singleQuote }) {
       let fixIt = FixIt(
         message: ReplaceTokensFixIt(replaceTokens: [singleQuote], replacements: [node.openQuote]),
         changes: [
@@ -1325,7 +1352,7 @@ public class ParseDiagnosticsGenerator: SyntaxAnyVisitor {
       addDiagnostic(diagnostic, handledNodes: handledNodes)
     }
     if case .stringSegment(let segment) = node.segments.last {
-      if let invalidContent = segment.unexpectedBeforeContent?.onlyToken(where: { $0.trailingTrivia.contains(where: { $0.isBackslash }) }) {
+      if let invalidContent = segment.unexpectedBeforeContent?.onlyPresentToken(where: { $0.trailingTrivia.contains(where: { $0.isBackslash }) }) {
         let fixIt = FixIt(
           message: .removeBackslash,
           changes: [
@@ -1350,7 +1377,7 @@ public class ParseDiagnosticsGenerator: SyntaxAnyVisitor {
       return .skipChildren
     }
     if let unexpected = node.unexpectedBetweenSubscriptKeywordAndGenericParameterClause,
-      let nameTokens = unexpected.onlyTokens(satisfying: { !$0.tokenKind.isLexerClassifiedKeyword })
+      let nameTokens = unexpected.onlyPresentTokens(satisfying: { !$0.tokenKind.isLexerClassifiedKeyword })
     {
       addDiagnostic(
         unexpected,
@@ -1362,7 +1389,7 @@ public class ParseDiagnosticsGenerator: SyntaxAnyVisitor {
       )
     }
     if let unexpected = node.indices.unexpectedBeforeLeftParen,
-      let nameTokens = unexpected.onlyTokens(satisfying: { !$0.tokenKind.isLexerClassifiedKeyword })
+      let nameTokens = unexpected.onlyPresentTokens(satisfying: { !$0.tokenKind.isLexerClassifiedKeyword })
     {
       addDiagnostic(
         unexpected,
@@ -1471,7 +1498,7 @@ public class ParseDiagnosticsGenerator: SyntaxAnyVisitor {
     }
 
     if let unexpected = node.unexpectedBeforeColon,
-      let leftParen = unexpected.onlyToken(where: { $0.tokenKind == .leftParen })
+      let leftParen = unexpected.onlyPresentToken(where: { $0.tokenKind == .leftParen })
     {
 
       var handledNodes: [SyntaxIdentifier] = [
@@ -1486,7 +1513,7 @@ public class ParseDiagnosticsGenerator: SyntaxAnyVisitor {
 
       var replaceTokens = [leftParen]
 
-      if let rightParen = node.unexpectedAfterInheritedTypeCollection?.onlyToken(where: { $0.tokenKind == .rightParen }) {
+      if let rightParen = node.unexpectedAfterInheritedTypeCollection?.onlyPresentToken(where: { $0.tokenKind == .rightParen }) {
         handledNodes += [rightParen.id]
         changes += [
           .makeMissing(rightParen)
@@ -1531,7 +1558,8 @@ public class ParseDiagnosticsGenerator: SyntaxAnyVisitor {
       return .skipChildren
     }
 
-    if let token = node.unexpectedBetweenMessageLabelAndColon?.onlyToken(where: { $0.tokenKind.isIdentifier }),
+    if let token = node.unexpectedBetweenMessageLabelAndColon?.onlyPresentToken(where: { $0.tokenKind.isIdentifier }),
+      token.presence == .present,
       node.messageLabel.presence == .missing
     {
       addDiagnostic(
@@ -1605,7 +1633,7 @@ public class ParseDiagnosticsGenerator: SyntaxAnyVisitor {
         }
 
         let unexpectedTokens: [TokenSyntax] = [detail.unexpectedBetweenLeftParenAndDetail, detail.unexpectedBetweenDetailAndRightParen]
-          .compactMap { $0?.tokens(viewMode: .all) }
+          .compactMap { $0?.tokens(viewMode: .sourceAccurate) }
           .flatMap { $0 }
 
         // If there is no unexpected tokens it means we miss a paren or set keyword.