Rename eof to endOfFile

CodeGeneration/Sources/SyntaxSupport/DeclNodes.swift

@@ -2033,7 +2033,7 @@ public let DECL_NODES: [Node] = [
     ]
   ),
 
-  // source-file = code-block-item-list eof
+  // source-file = code-block-item-list endOfFile
   Node(
     kind: .sourceFile,
     base: .syntax,
@@ -2048,7 +2048,7 @@ public let DECL_NODES: [Node] = [
       Child(
         name: "EndOfFileToken",
         deprecatedName: "EOFToken",
-        kind: .token(choices: [.token(tokenKind: "EOFToken")])
+        kind: .token(choices: [.token(tokenKind: "EndOfFileToken")])
       ),
     ]
   ),

CodeGeneration/Sources/SyntaxSupport/TokenSpec.swift

@@ -183,6 +183,7 @@ public let SYNTAX_TOKENS: [TokenSpec] = [
   PunctuatorSpec(name: "Comma", kind: "comma", text: ",", requiresTrailingSpace: true),
   MiscSpec(name: "DollarIdentifier", kind: "dollarident", nameForDiagnostics: "dollar identifier", classification: "DollarIdentifier"),
   PunctuatorSpec(name: "Ellipsis", kind: "ellipsis", text: "..."),
+  MiscSpec(name: "EndOfFile", kind: "eof", nameForDiagnostics: "end of file", text: ""),
   PunctuatorSpec(name: "Equal", kind: "equal", text: "=", requiresLeadingSpace: true, requiresTrailingSpace: true),
   PunctuatorSpec(name: "ExclamationMark", kind: "exclaim_postfix", text: "!"),
   MiscSpec(name: "ExtendedRegexDelimiter", kind: "extended_regex_delimiter", nameForDiagnostics: "extended delimiter", classification: "RegexLiteral"),
@@ -224,8 +225,6 @@ public let SYNTAX_TOKENS: [TokenSpec] = [
   MiscSpec(name: "Wildcard", kind: "_", nameForDiagnostics: "wildcard", text: "_"),
 ]
 
-// FIXME: Generate the EOF token as part of the normal SYNTAX_TOKENS and remove the special handling for it.
 public let SYNTAX_TOKEN_MAP = Dictionary(
-  uniqueKeysWithValues: (SYNTAX_TOKENS + [MiscSpec(name: "EOF", kind: "eof", nameForDiagnostics: "end of file", text: "")])
-    .map { ("\($0.name)Token", $0) }
+  uniqueKeysWithValues: SYNTAX_TOKENS.map { ("\($0.name)Token", $0) }
 )

CodeGeneration/Sources/Utils/SyntaxBuildableChild.swift

@@ -74,9 +74,6 @@ public extension Child {
     if token.isKeyword {
       return InitializerClauseSyntax(value: ExprSyntax(".\(raw: token.swiftKind)()"))
     }
-    if token.name == "EOF" {
-      return InitializerClauseSyntax(value: ExprSyntax(".eof()"))
-    }
     if token.text != nil {
       return InitializerClauseSyntax(value: ExprSyntax(".\(raw: token.swiftKind)Token()"))
     }

CodeGeneration/Sources/Utils/SyntaxBuildableType.swift

@@ -68,8 +68,6 @@ public struct SyntaxBuildableType: Hashable {
       } else if token.text != nil {
         return ExprSyntax(".\(raw: lowercaseFirstWord(name: token.name))Token()")
       }
-    } else if case .token("EOFToken") = kind {
-      return ExprSyntax(".eof()")
     }
     return nil
   }

CodeGeneration/Sources/generate-swiftsyntax/templates/ideutils/SyntaxClassificationFile.swift

@@ -77,13 +77,10 @@ let syntaxClassificationFile = SourceFileSyntax(leadingTrivia: copyrightHeader)
             if let classification = token.classification {
               StmtSyntax("return .\(raw: classification.swiftName)")
             } else {
-              StmtSyntax("return .none)")
+              StmtSyntax("return .none")
             }
           }
         }
-        SwitchCaseSyntax("case .eof:") {
-          StmtSyntax("return .none")
-        }
       }
     }
   }

CodeGeneration/Sources/generate-swiftsyntax/templates/swiftparser/IsLexerClassifiedFile.swift

@@ -58,10 +58,6 @@ let isLexerClassifiedFile = SourceFileSyntax(leadingTrivia: copyrightHeader) {
       """
     ) {
       try! SwitchExprSyntax("switch self") {
-        SwitchCaseSyntax("case .eof:") {
-          StmtSyntax("return false")
-        }
-
         for token in SYNTAX_TOKENS where token.isKeyword {
           SwitchCaseSyntax("case .\(raw: token.swiftKind):") {
             StmtSyntax("return true")

CodeGeneration/Sources/generate-swiftsyntax/templates/swiftparser/TokenSpecStaticMembersFile.swift

@@ -19,8 +19,6 @@ let tokenSpecStaticMembersFile = SourceFileSyntax(leadingTrivia: copyrightHeader
   DeclSyntax("@_spi(RawSyntax) import SwiftSyntax")
 
   try! ExtensionDeclSyntax("extension TokenSpec") {
-    DeclSyntax("static var eof: TokenSpec { return TokenSpec(.eof) }")
-
     for token in SYNTAX_TOKENS where token.swiftKind != "keyword" {
       DeclSyntax("static var \(raw: token.swiftKind): TokenSpec { return TokenSpec(.\(raw: token.swiftKind)) }")
     }

CodeGeneration/Sources/generate-swiftsyntax/templates/swiftparserdiagnostics/TokenNameForDiagnosticsFile.swift

@@ -21,10 +21,6 @@ let tokenNameForDiagnosticFile = SourceFileSyntax(leadingTrivia: copyrightHeader
   try! ExtensionDeclSyntax("extension TokenKind") {
     try! VariableDeclSyntax("var nameForDiagnostics: String") {
       try! SwitchExprSyntax("switch self") {
-        SwitchCaseSyntax("case .eof:") {
-          StmtSyntax(#"return "end of file""#)
-        }
-
         for token in SYNTAX_TOKENS where token.swiftKind != "keyword" {
           SwitchCaseSyntax("case .\(raw: token.swiftKind):") {
             StmtSyntax("return #\"\(raw: token.nameForDiagnostics)\"#")

CodeGeneration/Sources/generate-swiftsyntax/templates/swiftsyntax/TokenKindFile.swift

@@ -22,8 +22,6 @@ let tokenKindFile = SourceFileSyntax(leadingTrivia: copyrightHeader) {
     public enum TokenKind: Hashable
     """
   ) {
-    DeclSyntax("case eof")
-
     for token in SYNTAX_TOKENS {
       // Tokens that don't have a set text have an associated value that
       // contains their text.
@@ -59,10 +57,6 @@ let tokenKindFile = SourceFileSyntax(leadingTrivia: copyrightHeader) {
             }
           }
         }
-
-        SwitchCaseSyntax("case .eof:") {
-          StmtSyntax(#"return """#)
-        }
       }
     }
 
@@ -86,9 +80,6 @@ let tokenKindFile = SourceFileSyntax(leadingTrivia: copyrightHeader) {
           }
         }
 
-        SwitchCaseSyntax("case .eof:") {
-          StmtSyntax(#"return """#)
-        }
         SwitchCaseSyntax("default:") {
           StmtSyntax(#"return """#)
         }
@@ -106,10 +97,6 @@ let tokenKindFile = SourceFileSyntax(leadingTrivia: copyrightHeader) {
       """
     ) {
       try SwitchExprSyntax("switch self") {
-        SwitchCaseSyntax("case .eof:") {
-          StmtSyntax("return false")
-        }
-
         for token in SYNTAX_TOKENS {
           SwitchCaseSyntax("case .\(raw: token.swiftKind):") {
             StmtSyntax("return \(raw: type(of: token) == PunctuatorSpec.self)")
@@ -122,10 +109,6 @@ let tokenKindFile = SourceFileSyntax(leadingTrivia: copyrightHeader) {
   try! ExtensionDeclSyntax("extension TokenKind: Equatable") {
     try FunctionDeclSyntax("public static func ==(lhs: TokenKind, rhs: TokenKind) -> Bool") {
       try SwitchExprSyntax("switch (lhs, rhs)") {
-        SwitchCaseSyntax("case (.eof, .eof):") {
-          StmtSyntax("return true")
-        }
-
         for token in SYNTAX_TOKENS {
           if token.text != nil {
             SwitchCaseSyntax("case (.\(raw: token.swiftKind), .\(raw: token.swiftKind)):") {
@@ -156,8 +139,6 @@ let tokenKindFile = SourceFileSyntax(leadingTrivia: copyrightHeader) {
     public enum RawTokenKind: UInt8, Equatable, Hashable
     """
   ) {
-    DeclSyntax("case eof")
-
     for token in SYNTAX_TOKENS {
       DeclSyntax("case \(raw: token.swiftKind)")
     }
@@ -169,10 +150,6 @@ let tokenKindFile = SourceFileSyntax(leadingTrivia: copyrightHeader) {
       """
     ) {
       try! SwitchExprSyntax("switch self") {
-        SwitchCaseSyntax("case .eof:") {
-          StmtSyntax(#"return """#)
-        }
-
         for token in SYNTAX_TOKENS {
           if let text = token.text {
             SwitchCaseSyntax("case .\(raw: token.swiftKind):") {
@@ -198,10 +175,6 @@ let tokenKindFile = SourceFileSyntax(leadingTrivia: copyrightHeader) {
       """
     ) {
       try! SwitchExprSyntax("switch self") {
-        SwitchCaseSyntax("case .eof:") {
-          StmtSyntax("return false")
-        }
-
         for token in SYNTAX_TOKENS {
           SwitchCaseSyntax("case .\(raw: token.swiftKind):") {
             StmtSyntax("return \(raw: type(of: token) == PunctuatorSpec.self)")
@@ -220,10 +193,6 @@ let tokenKindFile = SourceFileSyntax(leadingTrivia: copyrightHeader) {
       """
     ) {
       try! SwitchExprSyntax("switch rawKind") {
-        SwitchCaseSyntax("case .eof:") {
-          StmtSyntax("return .eof")
-        }
-
         for token in SYNTAX_TOKENS {
           if token.swiftKind == "keyword" {
             SwitchCaseSyntax("case .\(raw: token.swiftKind):") {
@@ -259,10 +228,6 @@ let tokenKindFile = SourceFileSyntax(leadingTrivia: copyrightHeader) {
       """
     ) {
       try! SwitchExprSyntax("switch self") {
-        SwitchCaseSyntax("case .eof:") {
-          StmtSyntax("return (.eof, nil)")
-        }
-
         for token in SYNTAX_TOKENS {
           if token.swiftKind == "keyword" {
             SwitchCaseSyntax("case .\(raw: token.swiftKind)(let keyword):") {

CodeGeneration/Sources/generate-swiftsyntax/templates/swiftsyntax/TokensFile.swift

@@ -90,21 +90,5 @@ let tokensFile = SourceFileSyntax(leadingTrivia: copyrightHeader) {
         )
       }
     }
-
-    DeclSyntax(
-      """
-      public static func eof(
-        leadingTrivia: Trivia = [],
-        presence: SourcePresence = .present
-      ) -> TokenSyntax {
-        return TokenSyntax(
-          .eof,
-          leadingTrivia: leadingTrivia,
-          trailingTrivia: [],
-          presence: presence
-        )
-      }
-      """
-    )
   }
 }

CodeGeneration/Tests/ValidateSyntaxNodes/ValidateSyntaxNodes.swift

@@ -236,6 +236,8 @@ class ValidateSyntaxNodes: XCTestCase {
           message: "child 'ClosingPounds' has a token as its only token choice and should thus be named 'ExtendedRegexDelimiter'"
             // There are the opening and closing ExtendedRegexDelimiter in the node
         ),
+        // We should explicitly mention token here because it’s not obvious that the end of a file is represented by a token
+        ValidationFailure(node: .sourceFile, message: "child 'EndOfFileToken' has a token as its only token choice and should thus be named 'EndOfFile'"),
         ValidationFailure(
           node: .stringLiteralExpr,
           message: "child 'OpenDelimiter' has a token as its only token choice and should thus be named 'RawStringDelimiter'"
@@ -282,11 +284,6 @@ class ValidateSyntaxNodes: XCTestCase {
           node: .importPathComponent,
           message: "child 'TrailingPeriod' has a token as its only token choice and should thus be named 'Period'"
         ),
-        // We should explicitly mention token here because it’s not obvious that the end of a file is represented by a token
-        ValidationFailure(
-          node: .sourceFile,
-          message: "child 'EndOfFileToken' has a token as its only token choice and should thus be named 'EOF'"
-        ),
         // `~` is the only operator that’s allowed here
         ValidationFailure(
           node: .suppressedType,

Sources/SwiftBasicFormat/BasicFormat.swift

@@ -208,7 +208,7 @@ open class BasicFormat: SyntaxRewriter {
       (.backslash, _),
       (.backtick, _),
       (.dollarIdentifier, .period),  // a.b
-      (.eof, _),
+      (.endOfFile, _),
       (.exclamationMark, .leftParen),  // myOptionalClosure!()
       (.exclamationMark, .period),  // myOptionalBar!.foo()
       (.extendedRegexDelimiter, .leftParen),  // opening extended regex delimiter should never be separate by a space
@@ -254,7 +254,7 @@ open class BasicFormat: SyntaxRewriter {
       (.stringSegment, _),
       (_, .comma),
       (_, .ellipsis),
-      (_, .eof),
+      (_, .endOfFile),
       (_, .exclamationMark),
       (_, .postfixOperator),
       (_, .postfixQuestionMark),

Sources/SwiftIDEUtils/generated/SyntaxClassification.swift

@@ -122,6 +122,8 @@ extension RawTokenKind {
       return .dollarIdentifier
     case .ellipsis:
       return .none
+    case .endOfFile:
+      return .none
     case .equal:
       return .none
     case .exclamationMark:
@@ -200,8 +202,6 @@ extension RawTokenKind {
       return .none
     case .wildcard:
       return .none
-    case .eof:
-      return .none
     }
   }
 }

Sources/SwiftParser/Attributes.swift

@@ -287,7 +287,7 @@ extension Parser {
         // The contents of the @_effects attribute are parsed in SIL, we just
         // represent the contents as a list of tokens in SwiftSyntax.
         var tokens: [RawTokenSyntax] = []
-        while !parser.at(.rightParen, .eof) {
+        while !parser.at(.rightParen, .endOfFile) {
           tokens.append(parser.consumeAnyToken())
         }
         return .effectsArguments(RawEffectsArgumentsSyntax(elements: tokens, arena: parser.arena))
@@ -453,7 +453,7 @@ extension Parser {
 
     var elements = [RawDifferentiabilityParamSyntax]()
     var loopProgress = LoopProgressCondition()
-    while !self.at(.eof, .rightParen) && loopProgress.evaluate(currentToken) {
+    while !self.at(.endOfFile, .rightParen) && loopProgress.evaluate(currentToken) {
       guard let param = self.parseDifferentiabilityParameter() else {
         break
       }
@@ -676,7 +676,7 @@ extension Parser {
     var elements = [RawSpecializeAttributeSpecListSyntax.Element]()
     // Parse optional "exported" and "kind" labeled parameters.
     var loopProgress = LoopProgressCondition()
-    while !self.at(.eof, .rightParen, .keyword(.where)) && loopProgress.evaluate(currentToken) {
+    while !self.at(.endOfFile, .rightParen, .keyword(.where)) && loopProgress.evaluate(currentToken) {
       switch self.at(anyIn: SpecializeParameter.self) {
       case (.target, let handle)?:
         let ident = self.eat(handle)

Sources/SwiftParser/Declarations.swift

@@ -85,7 +85,7 @@ extension TokenConsumer {
     }
 
     if hasAttribute {
-      if subparser.at(.rightBrace) || subparser.at(.eof) || subparser.at(.poundEndifKeyword) {
+      if subparser.at(.rightBrace) || subparser.at(.endOfFile) || subparser.at(.poundEndifKeyword) {
         return true
       }
     }
@@ -776,7 +776,7 @@ extension Parser {
     let (unexpectedBeforeLBrace, lbrace) = self.expect(.leftBrace)
     do {
       var loopProgress = LoopProgressCondition()
-      while !self.at(.eof, .rightBrace) && loopProgress.evaluate(currentToken) {
+      while !self.at(.endOfFile, .rightBrace) && loopProgress.evaluate(currentToken) {
         let newItemAtStartOfLine = self.currentToken.isAtStartOfLine
         guard let newElement = self.parseMemberDeclListItem() else {
           break
@@ -1539,7 +1539,7 @@ extension Parser {
     var elements = [RawAccessorDeclSyntax]()
     do {
       var loopProgress = LoopProgressCondition()
-      while !self.at(.eof, .rightBrace) && loopProgress.evaluate(currentToken) {
+      while !self.at(.endOfFile, .rightBrace) && loopProgress.evaluate(currentToken) {
         guard let introducer = self.parseAccessorIntroducer() else {
           // There can only be an implicit getter if no other accessors were
           // seen before this one.
@@ -1758,7 +1758,7 @@ extension Parser {
     var loopProgress = LoopProgressCondition()
     while (identifiersAfterOperatorName.last ?? name).trailingTriviaByteLength == 0,
       self.currentToken.leadingTriviaByteLength == 0,
-      !self.at(.colon, .leftBrace, .eof),
+      !self.at(.colon, .leftBrace, .endOfFile),
       loopProgress.evaluate(self.currentToken)
     {
       identifiersAfterOperatorName.append(consumeAnyToken())
@@ -1906,7 +1906,7 @@ extension Parser {
     var elements = [RawPrecedenceGroupAttributeListSyntax.Element]()
     do {
       var attributesProgress = LoopProgressCondition()
-      LOOP: while !self.at(.eof, .rightBrace) && attributesProgress.evaluate(currentToken) {
+      LOOP: while !self.at(.endOfFile, .rightBrace) && attributesProgress.evaluate(currentToken) {
         switch self.at(anyIn: LabelText.self) {
         case (.associativity, let handle)?:
           let associativity = self.eat(handle)

Sources/SwiftParser/Directives.swift

@@ -205,7 +205,7 @@ extension Parser {
   ) -> [Element] {
     var elements = [Element]()
     var elementsProgress = LoopProgressCondition()
-    while !self.at(.eof)
+    while !self.at(.endOfFile)
       && !self.at(.poundElseKeyword, .poundElseifKeyword, .poundEndifKeyword)
       && !self.atElifTypo()
       && elementsProgress.evaluate(currentToken)