Parse shebang as a child of SourceFileSyntax

Author: StevenWong12

CodeGeneration/Sources/SyntaxSupport/DeclNodes.swift

@@ -1997,6 +1997,12 @@ public let DECL_NODES: [Node] = [
     parserFunction: "parseSourceFile",
     traits: ["WithStatements"],
     children: [
+      Child(
+        name: "Shebang",
+        kind: .token(choices: [.token(tokenKind: "ShebangToken")]),
+        documentation: "A shebang can specify the path of the compiler when using Swift source file as a script.",
+        isOptional: true
+      ),
       Child(
         name: "Statements",
         kind: .collection(kind: .codeBlockItemList, collectionElementName: "Statement")

CodeGeneration/Sources/SyntaxSupport/TokenSpec.swift

@@ -80,6 +80,7 @@ public let SYNTAX_TOKENS: [TokenSpec] = [
   .punctuator(name: "equal", text: "="),
   .punctuator(name: "exclamationMark", text: "!"),
   .other(name: "floatLiteral", nameForDiagnostics: "float literal"),
+  .other(name: "shebang", nameForDiagnostics: "shebang"),
   .other(name: "identifier", nameForDiagnostics: "identifier"),
   .punctuator(name: "infixQuestionMark", text: "?"),
   .other(name: "integerLiteral", nameForDiagnostics: "integer literal"),

CodeGeneration/Sources/SyntaxSupport/Trivia.swift

@@ -159,11 +159,6 @@ public let TRIVIAS: [Trivia] = [
     ]
   ),
 
-  Trivia(
-    name: "Shebang",
-    comment: #"A script command, starting with '#!'."#
-  ),
-
   Trivia(
     name: "Space",
     comment: #"A space ' ' character."#,

Sources/SwiftIDEUtils/SyntaxClassification.swift

@@ -193,6 +193,8 @@ extension RawTokenKind {
       return .none
     case .semicolon:
       return .none
+    case .shebang:
+      return .none
     case .singleQuote:
       return .stringLiteral
     case .stringQuote:

Sources/SwiftParser/Lexer/Cursor.swift

@@ -905,6 +905,11 @@ extension Lexer.Cursor {
     case UInt8(ascii: "\\"): _ = self.advance(); return Lexer.Result(.backslash)
 
     case UInt8(ascii: "#"):
+      // Try lex shebang.
+      if self.isAtStartOfFile, self.peek(at: 1) == UInt8(ascii: "!") {
+        self.advanceToEndOfLine()
+        return Lexer.Result(.shebang)
+      }
       // Try lex a raw string literal.
       if let delimiterLength = self.advanceIfOpeningRawStringDelimiter() {
         return Lexer.Result(.rawStringPoundDelimiter, stateTransition: .push(newState: .afterRawStringDelimiter(delimiterLength: delimiterLength)))
@@ -1172,11 +1177,7 @@ extension Lexer.Cursor {
           break
         }
       case UInt8(ascii: "#"):
-        guard start.isAtStartOfFile, self.advance(if: { $0 == "!" }) else {
-          break
-        }
-        self.advanceToEndOfLine()
-        continue
+        break
       case UInt8(ascii: "<"), UInt8(ascii: ">"):
         if self.tryLexConflictMarker(start: start) {
           error = LexingDiagnostic(.sourceConflictMarker, position: start)

Sources/SwiftParser/Lexer/RegexLiteralLexer.swift

@@ -625,6 +625,10 @@ extension Lexer.Cursor {
     case .arrow, .ellipsis, .period, .atSign, .pound, .backtick, .backslash:
       return false
 
+    // Shebang does not sequence expressions.
+    case .shebang:
+      return false
+
     case .keyword:
       // There are a handful of keywords that are expressions, handle them.
       // Otherwise, a regex literal can generally be parsed after a keyword.

Sources/SwiftParser/TokenPrecedence.swift

@@ -124,8 +124,8 @@ enum TokenPrecedence: Comparable {
       .dollarIdentifier, .identifier,
       // '_' can occur in types to replace a type identifier
       .wildcard,
-      // String segment, string interpolation anchor, pound, and regex pattern don't really fit anywhere else
-      .pound, .stringSegment, .regexLiteralPattern:
+      // String segment, string interpolation anchor, pound, shebang and regex pattern don't really fit anywhere else
+      .pound, .stringSegment, .regexLiteralPattern, .shebang:
       self = .identifierLike
 
     // MARK: Expr keyword

Sources/SwiftParser/TopLevel.swift

@@ -40,10 +40,12 @@ extension Parser {
   /// ``Parser/parse(source:parseTransition:filenameForDiagnostics:languageVersion:enableBareSlashRegexLiteral:)-7tndx``
   /// API calls.
   mutating func parseSourceFile() -> RawSourceFileSyntax {
+    let shebang = self.consume(if: .shebang)
     let items = self.parseTopLevelCodeBlockItems()
     let unexpectedBeforeEndOfFileToken = consumeRemainingTokens()
     let endOfFile = self.consume(if: .endOfFile)!
     return .init(
+      shebang: shebang,
       statements: items,
       RawUnexpectedNodesSyntax(unexpectedBeforeEndOfFileToken, arena: self.arena),
       endOfFileToken: endOfFile,

Sources/SwiftParser/TriviaParser.swift

@@ -81,15 +81,6 @@ public struct TriviaParser {
         }
 
       case UInt8(ascii: "#"):
-        // "#!...": .shebang
-        // NOTE: .shebang appears only if this trivia is at the start of the
-        // file. We don't know if this trivia is at the start of the file, but
-        // we believe that the lexer lexed it accordingly.
-        if position == .leading && pieces.isEmpty && cursor.advance(if: { $0 == "!" }) {
-          cursor.advanceToEndOfLine()
-          pieces.append(.shebang(start.text(upTo: cursor)))
-          continue
-        }
         cursor.advance(while: { $0 == "#" })
         pieces.append(.pounds(start.distance(to: cursor)))
         continue

Sources/SwiftParser/generated/TokenSpecStaticMembers.swift

@@ -67,6 +67,10 @@ extension TokenSpec {
     return TokenSpec(.floatLiteral)
   }
 
+  static var shebang: TokenSpec {
+    return TokenSpec(.shebang)
+  }
+  
   static var identifier: TokenSpec {
     return TokenSpec(.identifier)
   }

Sources/SwiftParserDiagnostics/generated/TokenNameForDiagnostics.swift

@@ -43,6 +43,8 @@ extension TokenKind {
       return "!"
     case .floatLiteral:
       return "float literal"
+    case .shebang:
+      return "shebang"
     case .identifier:
       return "identifier"
     case .infixQuestionMark:

Sources/SwiftSyntax/SourceLocation.swift

@@ -616,8 +616,7 @@ fileprivate extension RawTriviaPiece {
         body(carriageReturnLineLength)
       }
       lineLength = .zero
-    case let .shebang(text),
-      let .lineComment(text),
+    case let .lineComment(text),
       let .docLineComment(text):
       // Line comments are not supposed to contain newlines.
       precondition(!text.containsSwiftNewline(), "line comment created that contained a new-line character")

Sources/SwiftSyntax/Trivia.swift

@@ -192,7 +192,7 @@ extension Trivia {
         return Array(repeating: TriviaPiece.carriageReturns(1), count: count)
       case .carriageReturnLineFeeds(let count):
         return Array(repeating: TriviaPiece.carriageReturnLineFeeds(1), count: count)
-      case .lineComment, .blockComment, .docLineComment, .docBlockComment, .unexpectedText, .shebang:
+      case .lineComment, .blockComment, .docLineComment, .docBlockComment, .unexpectedText:
         return [piece]
       }
     })

Sources/SwiftSyntax/generated/ChildNameForKeyPath.swift

@@ -2779,8 +2779,12 @@ public func childName(_ keyPath: AnyKeyPath) -> String? {
     return "constraint"
   case \SomeOrAnyTypeSyntax.unexpectedAfterConstraint:
     return "unexpectedAfterConstraint"
-  case \SourceFileSyntax.unexpectedBeforeStatements:
-    return "unexpectedBeforeStatements"
+  case \SourceFileSyntax.unexpectedBeforeShebang:
+    return "unexpectedBeforeShebang"
+  case \SourceFileSyntax.shebang:
+    return "shebang"
+  case \SourceFileSyntax.unexpectedBetweenShebangAndStatements:
+    return "unexpectedBetweenShebangAndStatements"
   case \SourceFileSyntax.statements:
     return "statements"
   case \SourceFileSyntax.unexpectedBetweenStatementsAndEndOfFileToken:

Sources/SwiftSyntax/generated/RenamedChildrenCompatibility.swift

@@ -7056,11 +7056,13 @@ extension SourceFileSyntax {
     }
   }
 
-  @available(*, deprecated, renamed: "SourceFileSyntax(leadingTrivia:_:statements:_:endOfFileToken:_:trailingTrivia:)")
+  @available(*, deprecated, renamed: "SourceFileSyntax(leadingTrivia:_:shebang:_:statements:_:endOfFileToken:_:trailingTrivia:)")
   @_disfavoredOverload
   public init(
       leadingTrivia: Trivia? = nil,
-      _ unexpectedBeforeStatements: UnexpectedNodesSyntax? = nil,
+      _ unexpectedBeforeShebang: UnexpectedNodesSyntax? = nil,
+      shebang: TokenSyntax? = nil,
+      _ unexpectedBetweenShebangAndStatements: UnexpectedNodesSyntax? = nil,
       statements: CodeBlockItemListSyntax,
       _ unexpectedBetweenStatementsAndEOFToken: UnexpectedNodesSyntax? = nil,
       eofToken: TokenSyntax = .endOfFileToken(),
@@ -7070,7 +7072,9 @@ extension SourceFileSyntax {
   ) {
     self.init(
         leadingTrivia: leadingTrivia, 
-        unexpectedBeforeStatements, 
+        unexpectedBeforeShebang, 
+        shebang: shebang, 
+        unexpectedBetweenShebangAndStatements, 
         statements: statements, 
         unexpectedBetweenStatementsAndEOFToken, 
         endOfFileToken: eofToken, 

Sources/SwiftSyntax/generated/TokenKind.swift

@@ -27,6 +27,7 @@ public enum TokenKind: Hashable {
   case equal
   case exclamationMark
   case floatLiteral(String)
+  case shebang(String)
   case identifier(String)
   case infixQuestionMark
   case integerLiteral(String)
@@ -94,6 +95,8 @@ public enum TokenKind: Hashable {
       return #"!"#
     case .floatLiteral(let text):
       return text
+    case .shebang(let text):
+      return text
     case .identifier(let text):
       return text
     case .infixQuestionMark:
@@ -285,6 +288,8 @@ public enum TokenKind: Hashable {
       return true
     case .floatLiteral:
       return false
+    case .shebang:
+      return false
     case .identifier:
       return false
     case .infixQuestionMark:
@@ -390,6 +395,8 @@ extension TokenKind: Equatable {
       return true
     case (.floatLiteral(let lhsText), .floatLiteral(let rhsText)):
       return lhsText == rhsText
+    case (.shebang(let lhsText), .shebang(let rhsText)):
+      return lhsText == rhsText
     case (.identifier(let lhsText), .identifier(let rhsText)):
       return lhsText == rhsText
     case (.infixQuestionMark, .infixQuestionMark):
@@ -488,6 +495,7 @@ public enum RawTokenKind: UInt8, Equatable, Hashable {
   case equal
   case exclamationMark
   case floatLiteral
+  case shebang
   case identifier
   case infixQuestionMark
   case integerLiteral
@@ -638,6 +646,8 @@ public enum RawTokenKind: UInt8, Equatable, Hashable {
       return true
     case .floatLiteral:
       return false
+    case .shebang:
+      return false
     case .identifier:
       return false
     case .infixQuestionMark:
@@ -755,6 +765,8 @@ extension TokenKind {
       return .exclamationMark
     case .floatLiteral:
       return .floatLiteral(text)
+    case .shebang:
+      return .shebang(text)
     case .identifier:
       return .identifier(text)
     case .infixQuestionMark:
@@ -890,6 +902,8 @@ extension TokenKind {
       return (.exclamationMark, nil)
     case .floatLiteral(let str):
       return (.floatLiteral, str)
+    case .shebang(let str):
+      return (.shebang, str)
     case .identifier(let str):
       return (.identifier, str)
     case .infixQuestionMark:

Sources/SwiftSyntax/generated/Tokens.swift

@@ -201,6 +201,22 @@ extension TokenSyntax {
     )
   }
 
+  public static func shebang(
+    _ text: String,
+    leadingTrivia: Trivia = [],
+    trailingTrivia: Trivia = [],
+    presence: SourcePresence = .present
+    
+  ) -> TokenSyntax {
+    return TokenSyntax(
+      .shebang(text),
+      leadingTrivia: leadingTrivia,
+      trailingTrivia: trailingTrivia,
+      presence: presence
+      
+    )
+  }
+  
   public static func identifier(
     _ text: String,
     leadingTrivia: Trivia = [],

Sources/SwiftSyntax/generated/TriviaPieces.swift

@@ -41,8 +41,6 @@ public enum TriviaPiece {
   case newlines(Int)
   /// A '#' that is at the end of a line in a multi-line string literal to escape the newline.
   case pounds(Int)
-  /// A script command, starting with '#!'.
-  case shebang(String)
   /// A space ' ' character.
   case spaces(Int)
   /// A tab '\t' character.
@@ -84,8 +82,6 @@ extension TriviaPiece: TextOutputStreamable {
       printRepeated("\n", count: count)
     case let .pounds(count):
       printRepeated("#", count: count)
-    case let .shebang(text):
-      target.write(text)
     case let .spaces(count):
       printRepeated(" ", count: count)
     case let .tabs(count):
@@ -122,8 +118,6 @@ extension TriviaPiece: CustomDebugStringConvertible {
       return "newlines(\(data))"
     case .pounds(let data):
       return "pounds(\(data))"
-    case .shebang(let name):
-      return "shebang(\(name.debugDescription))"
     case .spaces(let data):
       return "spaces(\(data))"
     case .tabs(let data):
@@ -217,11 +211,6 @@ extension Trivia {
     return .pounds(1)
   }
 
-  /// Returns a piece of trivia for Shebang.
-  public static func shebang(_ text: String) -> Trivia {
-    return [.shebang(text)]
-  }
-  
   /// Returns a piece of trivia for some number of " " characters.
   public static func spaces(_ count: Int) -> Trivia {
     return [.spaces(count)]
@@ -283,8 +272,6 @@ extension TriviaPiece {
       return SourceLength(utf8Length: count)
     case let .pounds(count):
       return SourceLength(utf8Length: count)
-    case let .shebang(text):
-      return SourceLength(of: text)
     case let .spaces(count):
       return SourceLength(utf8Length: count)
     case let .tabs(count):
@@ -313,7 +300,6 @@ public enum RawTriviaPiece: Equatable {
   case lineComment(SyntaxText)
   case newlines(Int)
   case pounds(Int)
-  case shebang(SyntaxText)
   case spaces(Int)
   case tabs(Int)
   case unexpectedText(SyntaxText)
@@ -341,8 +327,6 @@ public enum RawTriviaPiece: Equatable {
       return .newlines(count)
     case let .pounds(count):
       return .pounds(count)
-    case let .shebang(text):
-      return .shebang(arena.intern(text))
     case let .spaces(count):
       return .spaces(count)
     case let .tabs(count):
@@ -378,8 +362,6 @@ extension TriviaPiece {
       self = .newlines(count)
     case let .pounds(count):
       self = .pounds(count)
-    case let .shebang(text):
-      self = .shebang(String(syntaxText: text))
     case let .spaces(count):
       self = .spaces(count)
     case let .tabs(count):
@@ -415,8 +397,6 @@ extension RawTriviaPiece {
       return count
     case let .pounds(count):
       return count
-    case let .shebang(text):
-      return text.count
     case let .spaces(count):
       return count
     case let .tabs(count):
@@ -450,8 +430,6 @@ extension RawTriviaPiece {
       return nil
     case .pounds(_):
       return nil
-    case .shebang(let text):
-      return text
     case .spaces(_):
       return nil
     case .tabs(_):