Run ParseDiagnosticGenerator on tree with flipped token presence

This uncovered a couple of implicit assumptions, mostly around the fact that tokens insided `UnexpectedNodesSyntax` are present, which isn’t true in general for manually generated trees.

It also uncovered an issue where we weren’t able to retrieve the trivia pieces of a token after it had been modified using `with` because after the modification the token was a `parsedToken` that no longer resided in a `ParsingSyntaxArena`.

Author: ahoppen

CodeGeneration/Sources/SyntaxSupport/DeclNodes.swift

@@ -560,19 +560,19 @@ public let DECL_NODES: [Node] = [
       Child(
         name: "Attributes",
         kind: .collection(kind: .attributeList, collectionElementName: "Attribute"),
-        description: "If there were attributes before the editor placeholder, the ``EditorPlaceholderDecl`` will contain these.",
+        documentation: "If there were attributes before the editor placeholder, the ``EditorPlaceholderDecl`` will contain these.",
         isOptional: true
       ),
       Child(
         name: "Modifiers",
         kind: .collection(kind: .modifierList, collectionElementName: "Modifier"),
-        description: "If there were modifiers before the editor placeholder, the `EditorPlaceholderDecl` will contain these.",
+        documentation: "If there were modifiers before the editor placeholder, the `EditorPlaceholderDecl` will contain these.",
         isOptional: true
       ),
       Child(
         name: "Placeholder",
         kind: .token(choices: [.token(tokenKind: "IdentifierToken")]),
-        description: """
+        documentation: """
           The actual editor placeholder that starts with `<#` and ends with `#>`.
           """
       ),

Sources/SwiftParserDiagnostics/MissingTokenError.swift

@@ -17,6 +17,7 @@ import SwiftDiagnostics
 extension ParseDiagnosticsGenerator {
   func handleMissingToken(_ missingToken: TokenSyntax) {
     guard let invalidToken = missingToken.previousToken(viewMode: .all),
+      invalidToken.presence == .present,
       let invalidTokenContainer = invalidToken.parent?.as(UnexpectedNodesSyntax.self),
       invalidTokenContainer.count == 1
     else {

Sources/SwiftParserDiagnostics/ParseDiagnosticsGenerator.swift

@@ -15,6 +15,26 @@ import SwiftDiagnostics
 @_spi(RawSyntax) import SwiftSyntax
 
 fileprivate func getTokens(between first: TokenSyntax, and second: TokenSyntax) -> [TokenSyntax] {
+  var first = first
+  if first.presence == .missing {
+    let nextPresentToken = first.nextToken(viewMode: .sourceAccurate)
+    guard let nextPresentToken else {
+      return []
+    }
+    first = nextPresentToken
+  }
+  precondition(first.presence == .present)
+
+  var second = second
+  if second.presence == .missing {
+    let previousPresentToken = second.previousToken(viewMode: .sourceAccurate)
+    guard let previousPresentToken else {
+      return []
+    }
+    second = previousPresentToken
+  }
+  precondition(second.presence == .present)
+
   var tokens: [TokenSyntax] = []
   var currentToken = first
 
@@ -149,7 +169,7 @@ public class ParseDiagnosticsGenerator: SyntaxAnyVisitor {
     removeRedundantFixIt: (_ misplacedTokens: [TokenSyntax]) -> FixItMessage? = { _ in nil }
   ) {
     guard let incorrectContainer = unexpected,
-      let misplacedTokens = incorrectContainer.onlyTokens(satisfying: unexpectedTokenCondition)
+      let misplacedTokens = incorrectContainer.onlyPresentTokens(satisfying: unexpectedTokenCondition)
     else {
       // If there are no unexpected nodes or the unexpected contain multiple tokens, don't emit a diagnostic.
       return
@@ -197,7 +217,7 @@ public class ParseDiagnosticsGenerator: SyntaxAnyVisitor {
     message: (TokenSyntax) -> some DiagnosticMessage
   ) {
     guard let unexpected = unexpected,
-      let misplacedToken = unexpected.onlyToken(where: predicate)
+      let misplacedToken = unexpected.onlyPresentToken(where: predicate)
     else {
       // If there is no unexpected node or the unexpected doesn't have the
       // expected token, don't emit a diagnostic.
@@ -279,7 +299,7 @@ public class ParseDiagnosticsGenerator: SyntaxAnyVisitor {
       }
       if specifier.presence == .present {
         for case .some(let unexpected) in unexpectedNodes {
-          for duplicateSpecifier in unexpected.tokens(satisfying: isOfSameKind) {
+          for duplicateSpecifier in unexpected.presentTokens(satisfying: isOfSameKind) {
             addDiagnostic(
               duplicateSpecifier,
               DuplicateEffectSpecifiers(correctSpecifier: specifier, unexpectedSpecifier: duplicateSpecifier),
@@ -315,29 +335,36 @@ public class ParseDiagnosticsGenerator: SyntaxAnyVisitor {
       suppressRemainingDiagnostics = true
       return .skipChildren
     }
-    if let tryKeyword = node.onlyToken(where: { $0.tokenKind == .keyword(.try) }),
+    if let tryKeyword = node.onlyPresentToken(where: { $0.tokenKind == .keyword(.try) }),
       let nextToken = tryKeyword.nextToken(viewMode: .sourceAccurate),
       nextToken.tokenKind.isLexerClassifiedKeyword,
       !(node.parent?.is(TypeEffectSpecifiersSyntax.self) ?? false)
     {
       addDiagnostic(node, TryCannotBeUsed(nextToken: nextToken))
-    } else if let semicolons = node.onlyTokens(satisfying: { $0.tokenKind == .semicolon }) {
+    } else if let semicolons = node.onlyPresentTokens(satisfying: { $0.tokenKind == .semicolon }) {
       addDiagnostic(
         node,
         .unexpectedSemicolon,
         fixIts: [
           FixIt(message: RemoveNodesFixIt(semicolons), changes: semicolons.map { FixIt.MultiNodeChange.makeMissing($0) })
         ]
       )
-    } else if node.first?.as(TokenSyntax.self)?.tokenKind.isIdentifier == true,
+    } else if let firstToken = node.first?.as(TokenSyntax.self),
+      firstToken.tokenKind.isIdentifier == true,
+      firstToken.presence == .present,
       let previousToken = node.previousToken(viewMode: .sourceAccurate),
       previousToken.tokenKind.isIdentifier,
       previousToken.parent?.is(DeclSyntax.self) == true || previousToken.parent?.is(IdentifierPatternSyntax.self) == true
     {
       // If multiple identifiers are used for a declaration name, offer to join them together.
       let tokens =
         node
-        .prefix(while: { $0.as(TokenSyntax.self)?.tokenKind.isIdentifier == true })
+        .prefix(while: {
+          guard let token = $0.as(TokenSyntax.self) else {
+            return false
+          }
+          return token.tokenKind.isIdentifier == true && token.presence == .present
+        })
         .map({ $0.as(TokenSyntax.self)! })
       let joined = previousToken.text + tokens.map(\.text).joined()
       var fixIts: [FixIt] = [
@@ -494,7 +521,7 @@ public class ParseDiagnosticsGenerator: SyntaxAnyVisitor {
     }
 
     if let unexpectedAfterRightParen = node.unexpectedAfterRightParen,
-      let (_, falseKeyword) = unexpectedAfterRightParen.twoTokens(
+      let (_, falseKeyword) = unexpectedAfterRightParen.twoPresentTokens(
         firstSatisfying: { $0.tokenKind == .binaryOperator("==") },
         secondSatisfying: { $0.tokenKind == .keyword(.false) }
       )
@@ -531,7 +558,7 @@ public class ParseDiagnosticsGenerator: SyntaxAnyVisitor {
       return .skipChildren
     }
     if let unexpected = node.unexpectedBetweenPlatformAndVersion,
-      unexpected.onlyToken(where: { $0.tokenKind == .binaryOperator(">=") }) != nil
+      unexpected.onlyPresentToken(where: { $0.tokenKind == .binaryOperator(">=") }) != nil
     {
       addDiagnostic(
         unexpected,
@@ -678,7 +705,7 @@ public class ParseDiagnosticsGenerator: SyntaxAnyVisitor {
     }
 
     if let unexpected = node.unexpectedBetweenBodyAndTrailingComma,
-      let token = unexpected.tokens(satisfying: { $0.tokenKind == .binaryOperator("&&") }).first,
+      let token = unexpected.presentTokens(satisfying: { $0.tokenKind == .binaryOperator("&&") }).first,
       let trailingComma = node.trailingComma,
       trailingComma.presence == .missing,
       let previous = node.unexpectedBetweenBodyAndTrailingComma?.previousToken(viewMode: .sourceAccurate)
@@ -709,7 +736,7 @@ public class ParseDiagnosticsGenerator: SyntaxAnyVisitor {
       return .skipChildren
     }
     if let unexpected = node.unexpectedBetweenDeinitKeywordAndBody,
-      let name = unexpected.filter({ $0.as(TokenSyntax.self)?.tokenKind.isIdentifier == true }).only?.as(TokenSyntax.self)
+      let name = unexpected.presentTokens(satisfying: { $0.tokenKind.isIdentifier == true }).only?.as(TokenSyntax.self)
     {
       addDiagnostic(
         name,
@@ -741,7 +768,7 @@ public class ParseDiagnosticsGenerator: SyntaxAnyVisitor {
       return .skipChildren
     }
     if node.floatingDigits.presence == .missing,
-      let (period, integerLiteral) = node.unexpectedAfterFloatingDigits?.twoTokens(
+      let (period, integerLiteral) = node.unexpectedAfterFloatingDigits?.twoPresentTokens(
         firstSatisfying: { $0.tokenKind == .period },
         secondSatisfying: { $0.tokenKind.isIntegerLiteral }
       )
@@ -772,7 +799,7 @@ public class ParseDiagnosticsGenerator: SyntaxAnyVisitor {
     // Detect C-style for loops based on two semicolons which could not be parsed between the 'for' keyword and the '{'
     // This is mostly a proof-of-concept implementation to produce more complex diagnostics.
     if let unexpectedCondition = node.body.unexpectedBeforeLeftBrace,
-      unexpectedCondition.tokens(withKind: .semicolon).count == 2
+      unexpectedCondition.presentTokens(withKind: .semicolon).count == 2
     {
       // FIXME: This is aweful. We should have a way to either get all children between two cursors in a syntax node or highlight a range from one node to another.
       addDiagnostic(
@@ -855,7 +882,7 @@ public class ParseDiagnosticsGenerator: SyntaxAnyVisitor {
         message: { _ in .typeParameterPackEllipsis }
       )
     } else if let unexpected = node.unexpectedBetweenNameAndColon,
-      let unexpectedEllipsis = unexpected.onlyToken(where: { $0.tokenKind == .ellipsis }),
+      let unexpectedEllipsis = unexpected.onlyPresentToken(where: { $0.tokenKind == .ellipsis }),
       let each = node.each
     {
       addDiagnostic(
@@ -1105,7 +1132,7 @@ public class ParseDiagnosticsGenerator: SyntaxAnyVisitor {
     if shouldSkip(node) {
       return .skipChildren
     }
-    if let token = node.unexpectedBetweenModuleLabelAndColon?.onlyToken(where: { $0.tokenKind.isIdentifier }),
+    if let token = node.unexpectedBetweenModuleLabelAndColon?.onlyPresentToken(where: { $0.tokenKind.isIdentifier }),
       node.moduleLabel.presence == .missing
     {
       addDiagnostic(
@@ -1172,9 +1199,9 @@ public class ParseDiagnosticsGenerator: SyntaxAnyVisitor {
         return
       }
       let message: DiagnosticMessage?
-      if let identifier = unexpected.onlyToken(where: { $0.tokenKind.isIdentifier }) {
+      if let identifier = unexpected.onlyPresentToken(where: { $0.tokenKind.isIdentifier }) {
         message = IdentifierNotAllowedInOperatorName(identifier: identifier)
-      } else if let tokens = unexpected.onlyTokens(satisfying: { _ in true }) {
+      } else if let tokens = unexpected.onlyPresentTokens(satisfying: { _ in true }) {
         message = TokensNotAllowedInOperatorName(tokens: tokens)
       } else {
         message = nil
@@ -1265,7 +1292,7 @@ public class ParseDiagnosticsGenerator: SyntaxAnyVisitor {
       return .skipChildren
     }
     // recover from Objective-C style literals
-    if let atSign = node.unexpectedBetweenOpenDelimiterAndOpenQuote?.onlyToken(where: { $0.tokenKind == .atSign }) {
+    if let atSign = node.unexpectedBetweenOpenDelimiterAndOpenQuote?.onlyPresentToken(where: { $0.tokenKind == .atSign }) {
       addDiagnostic(
         node,
         .stringLiteralAtSign,
@@ -1275,7 +1302,7 @@ public class ParseDiagnosticsGenerator: SyntaxAnyVisitor {
         handledNodes: [atSign.id]
       )
     }
-    if let singleQuote = node.unexpectedBetweenOpenDelimiterAndOpenQuote?.onlyToken(where: { $0.tokenKind == .singleQuote }) {
+    if let singleQuote = node.unexpectedBetweenOpenDelimiterAndOpenQuote?.onlyPresentToken(where: { $0.tokenKind == .singleQuote }) {
       let fixIt = FixIt(
         message: ReplaceTokensFixIt(replaceTokens: [singleQuote], replacements: [node.openQuote]),
         changes: [
@@ -1325,7 +1352,7 @@ public class ParseDiagnosticsGenerator: SyntaxAnyVisitor {
       addDiagnostic(diagnostic, handledNodes: handledNodes)
     }
     if case .stringSegment(let segment) = node.segments.last {
-      if let invalidContent = segment.unexpectedBeforeContent?.onlyToken(where: { $0.trailingTrivia.contains(where: { $0.isBackslash }) }) {
+      if let invalidContent = segment.unexpectedBeforeContent?.onlyPresentToken(where: { $0.trailingTrivia.contains(where: { $0.isBackslash }) }) {
         let fixIt = FixIt(
           message: .removeBackslash,
           changes: [
@@ -1350,7 +1377,7 @@ public class ParseDiagnosticsGenerator: SyntaxAnyVisitor {
       return .skipChildren
     }
     if let unexpected = node.unexpectedBetweenSubscriptKeywordAndGenericParameterClause,
-      let nameTokens = unexpected.onlyTokens(satisfying: { !$0.tokenKind.isLexerClassifiedKeyword })
+      let nameTokens = unexpected.onlyPresentTokens(satisfying: { !$0.tokenKind.isLexerClassifiedKeyword })
     {
       addDiagnostic(
         unexpected,
@@ -1362,7 +1389,7 @@ public class ParseDiagnosticsGenerator: SyntaxAnyVisitor {
       )
     }
     if let unexpected = node.indices.unexpectedBeforeLeftParen,
-      let nameTokens = unexpected.onlyTokens(satisfying: { !$0.tokenKind.isLexerClassifiedKeyword })
+      let nameTokens = unexpected.onlyPresentTokens(satisfying: { !$0.tokenKind.isLexerClassifiedKeyword })
     {
       addDiagnostic(
         unexpected,
@@ -1471,7 +1498,7 @@ public class ParseDiagnosticsGenerator: SyntaxAnyVisitor {
     }
 
     if let unexpected = node.unexpectedBeforeColon,
-      let leftParen = unexpected.onlyToken(where: { $0.tokenKind == .leftParen })
+      let leftParen = unexpected.onlyPresentToken(where: { $0.tokenKind == .leftParen })
     {
 
       var handledNodes: [SyntaxIdentifier] = [
@@ -1486,7 +1513,7 @@ public class ParseDiagnosticsGenerator: SyntaxAnyVisitor {
 
       var replaceTokens = [leftParen]
 
-      if let rightParen = node.unexpectedAfterInheritedTypeCollection?.onlyToken(where: { $0.tokenKind == .rightParen }) {
+      if let rightParen = node.unexpectedAfterInheritedTypeCollection?.onlyPresentToken(where: { $0.tokenKind == .rightParen }) {
         handledNodes += [rightParen.id]
         changes += [
           .makeMissing(rightParen)
@@ -1531,7 +1558,8 @@ public class ParseDiagnosticsGenerator: SyntaxAnyVisitor {
       return .skipChildren
     }
 
-    if let token = node.unexpectedBetweenMessageLabelAndColon?.onlyToken(where: { $0.tokenKind.isIdentifier }),
+    if let token = node.unexpectedBetweenMessageLabelAndColon?.onlyPresentToken(where: { $0.tokenKind.isIdentifier }),
+      token.presence == .present,
       node.messageLabel.presence == .missing
     {
       addDiagnostic(
@@ -1605,7 +1633,7 @@ public class ParseDiagnosticsGenerator: SyntaxAnyVisitor {
         }
 
         let unexpectedTokens: [TokenSyntax] = [detail.unexpectedBetweenLeftParenAndDetail, detail.unexpectedBetweenDetailAndRightParen]
-          .compactMap { $0?.tokens(viewMode: .all) }
+          .compactMap { $0?.tokens(viewMode: .sourceAccurate) }
           .flatMap { $0 }
 
         // If there is no unexpected tokens it means we miss a paren or set keyword.

Sources/SwiftParserDiagnostics/SyntaxExtensions.swift

@@ -14,36 +14,40 @@
 @_spi(Diagnostics) import SwiftParser
 
 extension UnexpectedNodesSyntax {
-  func tokens(satisfying isIncluded: (TokenSyntax) -> Bool) -> [TokenSyntax] {
+  func presentTokens(satisfying isIncluded: (TokenSyntax) -> Bool) -> [TokenSyntax] {
     return self.children(viewMode: .sourceAccurate).compactMap({ $0.as(TokenSyntax.self) }).filter(isIncluded)
   }
 
-  func tokens(withKind kind: TokenKind) -> [TokenSyntax] {
-    return self.tokens(satisfying: { $0.tokenKind == kind })
+  func presentTokens(withKind kind: TokenKind) -> [TokenSyntax] {
+    return self.presentTokens(satisfying: { $0.tokenKind == kind })
   }
 
-  /// If this only contains a single item, which is a token satisfying `condition`, return that token, otherwise return `nil`.
-  func onlyToken(where condition: (TokenSyntax) -> Bool) -> TokenSyntax? {
-    if self.count == 1, let token = self.first?.as(TokenSyntax.self), condition(token) {
+  /// If this only contains a single item, which is a present token satisfying `condition`, return that token, otherwise return `nil`.
+  func onlyPresentToken(where condition: (TokenSyntax) -> Bool) -> TokenSyntax? {
+    if self.count == 1,
+      let token = self.first?.as(TokenSyntax.self),
+      condition(token),
+      token.presence == .present
+    {
       return token
     } else {
       return nil
     }
   }
 
-  /// If this only contains tokens satisfying `condition`, return an array containing those tokens, otherwise return `nil`.
-  func onlyTokens(satisfying condition: (TokenSyntax) -> Bool) -> [TokenSyntax]? {
-    let tokens = tokens(satisfying: condition)
+  /// If this only contains present tokens satisfying `condition`, return an array containing those tokens, otherwise return `nil`.
+  func onlyPresentTokens(satisfying condition: (TokenSyntax) -> Bool) -> [TokenSyntax]? {
+    let tokens = presentTokens(satisfying: condition)
     if tokens.count == self.count {
       return tokens
     } else {
       return nil
     }
   }
 
-  /// If this only contains two tokens, the first satisfying `firstCondition`, and the second satisfying `secondCondition`,
+  /// If this only contains two present tokens, the first satisfying `firstCondition`, and the second satisfying `secondCondition`,
   /// return these tokens as a tuple, otherwise return `nil`.
-  func twoTokens(
+  func twoPresentTokens(
     firstSatisfying firstCondition: (TokenSyntax) -> Bool,
     secondSatisfying secondCondition: (TokenSyntax) -> Bool
   ) -> (first: TokenSyntax, second: TokenSyntax)? {
@@ -95,7 +99,7 @@ extension SyntaxProtocol {
       } else {
         return "braces"
       }
-    } else if let token = Syntax(self).as(UnexpectedNodesSyntax.self)?.onlyTokens(satisfying: { $0.tokenKind.isLexerClassifiedKeyword })?.only {
+    } else if let token = Syntax(self).as(UnexpectedNodesSyntax.self)?.onlyPresentTokens(satisfying: { $0.tokenKind.isLexerClassifiedKeyword })?.only {
       return "'\(token.text)' keyword"
     } else if let token = Syntax(self).as(TokenSyntax.self) {
       return "'\(token.text)' keyword"

Sources/SwiftSyntax/Raw/RawSyntaxTokenView.swift

@@ -95,7 +95,7 @@ public struct RawSyntaxTokenView {
   public var leadingRawTriviaPieces: [RawTriviaPiece] {
     switch raw.rawData.payload {
     case .parsedToken(let dat):
-      let arena = raw.arena as! ParsingSyntaxArena
+      let arena = raw.arena.parsingArena!
       return arena.parseTrivia(source: dat.leadingTriviaText, position: .leading)
     case .materializedToken(let dat):
       return Array(dat.leadingTrivia)
@@ -108,7 +108,7 @@ public struct RawSyntaxTokenView {
   public var trailingRawTriviaPieces: [RawTriviaPiece] {
     switch raw.rawData.payload {
     case .parsedToken(let dat):
-      let arena = raw.arena as! ParsingSyntaxArena
+      let arena = raw.arena.parsingArena!
       return arena.parseTrivia(source: dat.trailingTriviaText, position: .trailing)
     case .materializedToken(let dat):
       return Array(dat.trailingTrivia)