From 35e2406cf1f400327555917ecd4180de7d4e740c Mon Sep 17 00:00:00 2001 From: stevenwong Date: Thu, 2 Mar 2023 18:06:12 +0800 Subject: [PATCH 01/12] Check if arrow or leftbrace can be recovered --- Sources/SwiftParser/Declarations.swift | 2 +- Sources/SwiftParser/TopLevel.swift | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Sources/SwiftParser/Declarations.swift b/Sources/SwiftParser/Declarations.swift index 310b95b379d..be9e417ddc7 100644 --- a/Sources/SwiftParser/Declarations.swift +++ b/Sources/SwiftParser/Declarations.swift @@ -1375,7 +1375,7 @@ extension Parser { var effectSpecifiers = self.parseDeclEffectSpecifiers() let output: RawReturnClauseSyntax? - if self.at(.arrow) { + if let _ = self.canRecoverTo(TokenSpec(.arrow)) { output = self.parseFunctionReturnClause(effectSpecifiers: &effectSpecifiers, allowNamedOpaqueResultType: true) } else { output = nil diff --git a/Sources/SwiftParser/TopLevel.swift b/Sources/SwiftParser/TopLevel.swift index 7f3899cb154..ea85f31fb15 100644 --- a/Sources/SwiftParser/TopLevel.swift +++ b/Sources/SwiftParser/TopLevel.swift @@ -98,7 +98,7 @@ extension Parser { /// This function is used when parsing places where function bodies are /// optional - like the function requirements in protocol declarations. mutating func parseOptionalCodeBlock(allowInitDecl: Bool = true) -> RawCodeBlockSyntax? { - guard self.at(.leftBrace) else { + guard let _ = self.canRecoverTo(TokenSpec(.leftBrace)) else { return nil } return self.parseCodeBlock(allowInitDecl: allowInitDecl) From 3c7a970d9cd46958ec8e8a9017713bcfc743662b Mon Sep 17 00:00:00 2001 From: stevenwong Date: Thu, 2 Mar 2023 18:07:18 +0800 Subject: [PATCH 02/12] Modify the init function of TokenPrecedence --- Sources/SwiftParser/TokenPrecedence.swift | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/Sources/SwiftParser/TokenPrecedence.swift b/Sources/SwiftParser/TokenPrecedence.swift index 9701b62cd00..6c130c907b7 100644 --- a/Sources/SwiftParser/TokenPrecedence.swift +++ b/Sources/SwiftParser/TokenPrecedence.swift @@ -101,8 +101,8 @@ public enum TokenPrecedence: Comparable { @_spi(RawSyntax) public init(_ lexeme: Lexer.Lexeme) { - if lexeme.rawTokenKind == .keyword { - self.init(Keyword(lexeme.tokenText)!) + if let kw = Keyword(lexeme.tokenText) { + self.init(kw) } else { self.init(nonKeyword: lexeme.rawTokenKind) } @@ -216,7 +216,9 @@ public enum TokenPrecedence: Comparable { // Access modifiers .fileprivate, .internal, .private, .public, .static, // Functions - .deinit, .func, .`init`, .subscript, + .deinit, .func, .`init`, .subscript, + // Marco + .macro, // Variables .let, .var, // Operator stuff From 24fc5ea3f8c4fa737bb47f76a05fee0e2f981039 Mon Sep 17 00:00:00 2001 From: stevenwong Date: Thu, 2 Mar 2023 18:08:24 +0800 Subject: [PATCH 03/12] Adapt test cases to the change --- Tests/SwiftParserTest/DeclarationTests.swift | 2 +- .../translated/EffectfulPropertiesTests.swift | 4 ++-- Tests/SwiftParserTest/translated/InvalidTests.swift | 5 ++++- Tests/SwiftParserTest/translated/RecoveryTests.swift | 11 ++++++----- 4 files changed, 13 insertions(+), 9 deletions(-) diff --git a/Tests/SwiftParserTest/DeclarationTests.swift b/Tests/SwiftParserTest/DeclarationTests.swift index 628ee4d159c..78b60525a6e 100644 --- a/Tests/SwiftParserTest/DeclarationTests.swift +++ b/Tests/SwiftParserTest/DeclarationTests.swift @@ -917,7 +917,7 @@ final class DeclarationTests: XCTestCase { diagnostics: [ DiagnosticSpec(locationMarker: "1️⃣", message: "expected ':' in parameter"), DiagnosticSpec(locationMarker: "2️⃣", message: "expected ')' to end parameter clause"), - DiagnosticSpec(locationMarker: "3️⃣", message: "extraneous code ': Int) {}' at top level"), + DiagnosticSpec(locationMarker: "3️⃣", message: "unexpected code ': Int)' in function"), ] ) } diff --git a/Tests/SwiftParserTest/translated/EffectfulPropertiesTests.swift b/Tests/SwiftParserTest/translated/EffectfulPropertiesTests.swift index 9ef8041c0cf..14d4809583f 100644 --- a/Tests/SwiftParserTest/translated/EffectfulPropertiesTests.swift +++ b/Tests/SwiftParserTest/translated/EffectfulPropertiesTests.swift @@ -231,7 +231,7 @@ final class EffectfulPropertiesTests: XCTestCase { } """, diagnostics: [ - DiagnosticSpec(message: "unexpected code 'bogus {}' in variable") + DiagnosticSpec(message: "unexpected code 'bogus' in accessor") ] ) } @@ -257,7 +257,7 @@ final class EffectfulPropertiesTests: XCTestCase { } """, diagnostics: [ - DiagnosticSpec(message: "unexpected code '-> Int { 0 }' in variable") + DiagnosticSpec(message: "unexpected code '-> Int' in accessor") ] ) } diff --git a/Tests/SwiftParserTest/translated/InvalidTests.swift b/Tests/SwiftParserTest/translated/InvalidTests.swift index c81b8e5c172..76a8a5432f8 100644 --- a/Tests/SwiftParserTest/translated/InvalidTests.swift +++ b/Tests/SwiftParserTest/translated/InvalidTests.swift @@ -205,7 +205,8 @@ final class InvalidTests: XCTestCase { DiagnosticSpec(locationMarker: "2️⃣", message: "expected ')' in function type"), DiagnosticSpec(locationMarker: "3️⃣", message: "expected return type in function type"), DiagnosticSpec(locationMarker: "3️⃣", message: "expected ')' to end parameter clause"), - DiagnosticSpec(locationMarker: "4️⃣", message: "extraneous code ') {}' at top level"), + DiagnosticSpec(locationMarker: "3️⃣", message: "unexpected code '()' in function signature"), + DiagnosticSpec(locationMarker: "4️⃣", message: "unexpected code ')' in function") ] ) } @@ -484,6 +485,7 @@ final class InvalidTests: XCTestCase { diagnostics: [ DiagnosticSpec(locationMarker: "1️⃣", message: "expected '(' to start parameter clause"), DiagnosticSpec(locationMarker: "2️⃣", message: "expected ')' to end parameter clause"), + DiagnosticSpec(locationMarker: "2️⃣", message: "unexpected code '()' in function") ] ) } @@ -497,6 +499,7 @@ final class InvalidTests: XCTestCase { diagnostics: [ DiagnosticSpec(locationMarker: "1️⃣", message: "expected '(' to start parameter clause"), DiagnosticSpec(locationMarker: "2️⃣", message: "expected ')' to end parameter clause"), + DiagnosticSpec(locationMarker: "2️⃣", message: "unexpected code '(x: T)' in function") ] ) } diff --git a/Tests/SwiftParserTest/translated/RecoveryTests.swift b/Tests/SwiftParserTest/translated/RecoveryTests.swift index dfd0343be20..45b3d11428a 100644 --- a/Tests/SwiftParserTest/translated/RecoveryTests.swift +++ b/Tests/SwiftParserTest/translated/RecoveryTests.swift @@ -1206,13 +1206,14 @@ final class RecoveryTests: XCTestCase { func bar() -> Int3️⃣] { return [0] } - } + 4️⃣} """, diagnostics: [ DiagnosticSpec(locationMarker: "1️⃣", message: "expected '}' to end struct"), DiagnosticSpec(locationMarker: "2️⃣", message: "expected ']' to end array"), // TODO: Old parser expected error on line 5: unexpected ']' in type; did you mean to write an array type?, Fix-It replacements: 17 - 17 = '[' - DiagnosticSpec(locationMarker: "3️⃣", message: "extraneous code at top level"), + DiagnosticSpec(locationMarker: "3️⃣", message: "unexpected code ']' in function"), + DiagnosticSpec(locationMarker: "4️⃣", message: "extraneous brace at top level"), ] ) } @@ -1247,7 +1248,7 @@ final class RecoveryTests: XCTestCase { """, diagnostics: [ // TODO: Old parser expected error on line 2: array types are now written with the brackets around the element type, Fix-It replacements: 17 - 17 = '[', 20 - 21 = '' - DiagnosticSpec(message: "unexpected code in struct") + DiagnosticSpec(message: "unexpected code '[0]' in function") ] ) } @@ -1263,7 +1264,7 @@ final class RecoveryTests: XCTestCase { """, diagnostics: [ // TODO: Old parser expected error on line 2: array types are now written with the brackets around the element type, Fix-It replacements: 17 - 17 = '[', 20 - 21 = '' - DiagnosticSpec(message: "unexpected code in struct") + DiagnosticSpec(message: "unexpected code '[0_1]' in function") ] ) } @@ -1279,7 +1280,7 @@ final class RecoveryTests: XCTestCase { """, diagnostics: [ // TODO: Old parser expected error on line 2: array types are now written with the brackets around the element type, Fix-It replacements: 17 - 17 = '[', 20 - 21 = '' - DiagnosticSpec(message: "unexpected code in struct") + DiagnosticSpec(message: "unexpected code '[0b1]' in function") ] ) } From e0fd4b61b2554b994196c2125b8e4fd95a5137cc Mon Sep 17 00:00:00 2001 From: stevenwong Date: Thu, 2 Mar 2023 18:14:03 +0800 Subject: [PATCH 04/12] Add new test case for recovery --- .../translated/RecoveryTests.swift | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/Tests/SwiftParserTest/translated/RecoveryTests.swift b/Tests/SwiftParserTest/translated/RecoveryTests.swift index 45b3d11428a..2a0c360d912 100644 --- a/Tests/SwiftParserTest/translated/RecoveryTests.swift +++ b/Tests/SwiftParserTest/translated/RecoveryTests.swift @@ -2206,4 +2206,20 @@ final class RecoveryTests: XCTestCase { ) } + func testRecovery182() { + AssertParse( + "func foo() 1️⃣bogus {}", + diagnostics: [ + DiagnosticSpec(message: "unexpected code 'bogus' in function") + ] + ) + + AssertParse( + "func foo() 1️⃣bogus -> Int {}", + diagnostics: [ + DiagnosticSpec(message: "unexpected code 'bogus' in function signature") + ] + ) + } + } From 68f587995ab0f6538f8f4f5b6953a70f954e5605 Mon Sep 17 00:00:00 2001 From: stevenwong Date: Thu, 2 Mar 2023 19:45:26 +0800 Subject: [PATCH 05/12] Format codes with swift-format --- Sources/SwiftParser/TokenPrecedence.swift | 2 +- Tests/SwiftParserTest/translated/InvalidTests.swift | 6 +++--- Tests/SwiftParserTest/translated/RecoveryTests.swift | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/Sources/SwiftParser/TokenPrecedence.swift b/Sources/SwiftParser/TokenPrecedence.swift index 6c130c907b7..97f0538ddad 100644 --- a/Sources/SwiftParser/TokenPrecedence.swift +++ b/Sources/SwiftParser/TokenPrecedence.swift @@ -216,7 +216,7 @@ public enum TokenPrecedence: Comparable { // Access modifiers .fileprivate, .internal, .private, .public, .static, // Functions - .deinit, .func, .`init`, .subscript, + .deinit, .func, .`init`, .subscript, // Marco .macro, // Variables diff --git a/Tests/SwiftParserTest/translated/InvalidTests.swift b/Tests/SwiftParserTest/translated/InvalidTests.swift index 76a8a5432f8..267e1ced6f6 100644 --- a/Tests/SwiftParserTest/translated/InvalidTests.swift +++ b/Tests/SwiftParserTest/translated/InvalidTests.swift @@ -206,7 +206,7 @@ final class InvalidTests: XCTestCase { DiagnosticSpec(locationMarker: "3️⃣", message: "expected return type in function type"), DiagnosticSpec(locationMarker: "3️⃣", message: "expected ')' to end parameter clause"), DiagnosticSpec(locationMarker: "3️⃣", message: "unexpected code '()' in function signature"), - DiagnosticSpec(locationMarker: "4️⃣", message: "unexpected code ')' in function") + DiagnosticSpec(locationMarker: "4️⃣", message: "unexpected code ')' in function"), ] ) } @@ -485,7 +485,7 @@ final class InvalidTests: XCTestCase { diagnostics: [ DiagnosticSpec(locationMarker: "1️⃣", message: "expected '(' to start parameter clause"), DiagnosticSpec(locationMarker: "2️⃣", message: "expected ')' to end parameter clause"), - DiagnosticSpec(locationMarker: "2️⃣", message: "unexpected code '()' in function") + DiagnosticSpec(locationMarker: "2️⃣", message: "unexpected code '()' in function"), ] ) } @@ -499,7 +499,7 @@ final class InvalidTests: XCTestCase { diagnostics: [ DiagnosticSpec(locationMarker: "1️⃣", message: "expected '(' to start parameter clause"), DiagnosticSpec(locationMarker: "2️⃣", message: "expected ')' to end parameter clause"), - DiagnosticSpec(locationMarker: "2️⃣", message: "unexpected code '(x: T)' in function") + DiagnosticSpec(locationMarker: "2️⃣", message: "unexpected code '(x: T)' in function"), ] ) } diff --git a/Tests/SwiftParserTest/translated/RecoveryTests.swift b/Tests/SwiftParserTest/translated/RecoveryTests.swift index 2a0c360d912..6fb970f456c 100644 --- a/Tests/SwiftParserTest/translated/RecoveryTests.swift +++ b/Tests/SwiftParserTest/translated/RecoveryTests.swift @@ -1212,7 +1212,7 @@ final class RecoveryTests: XCTestCase { DiagnosticSpec(locationMarker: "1️⃣", message: "expected '}' to end struct"), DiagnosticSpec(locationMarker: "2️⃣", message: "expected ']' to end array"), // TODO: Old parser expected error on line 5: unexpected ']' in type; did you mean to write an array type?, Fix-It replacements: 17 - 17 = '[' - DiagnosticSpec(locationMarker: "3️⃣", message: "unexpected code ']' in function"), + DiagnosticSpec(locationMarker: "3️⃣", message: "unexpected code ']' in function"), DiagnosticSpec(locationMarker: "4️⃣", message: "extraneous brace at top level"), ] ) From 1526974fe331f0213514b04b8467883f7b9957e1 Mon Sep 17 00:00:00 2001 From: stevenwong Date: Thu, 2 Mar 2023 22:01:35 +0800 Subject: [PATCH 06/12] Fix a typo in comment --- Sources/SwiftParser/TokenPrecedence.swift | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Sources/SwiftParser/TokenPrecedence.swift b/Sources/SwiftParser/TokenPrecedence.swift index 97f0538ddad..8fefebe7875 100644 --- a/Sources/SwiftParser/TokenPrecedence.swift +++ b/Sources/SwiftParser/TokenPrecedence.swift @@ -217,7 +217,7 @@ public enum TokenPrecedence: Comparable { .fileprivate, .internal, .private, .public, .static, // Functions .deinit, .func, .`init`, .subscript, - // Marco + // Macro .macro, // Variables .let, .var, From 5e91be29b4ff6c07de31d9fa52d9bcaf0fdfb19c Mon Sep 17 00:00:00 2001 From: stevenwong Date: Fri, 3 Mar 2023 12:31:48 +0800 Subject: [PATCH 07/12] Revert the changes in TokenPrecedence --- Sources/SwiftParser/TokenPrecedence.swift | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Sources/SwiftParser/TokenPrecedence.swift b/Sources/SwiftParser/TokenPrecedence.swift index 8fefebe7875..788ec1ef6f7 100644 --- a/Sources/SwiftParser/TokenPrecedence.swift +++ b/Sources/SwiftParser/TokenPrecedence.swift @@ -101,8 +101,8 @@ public enum TokenPrecedence: Comparable { @_spi(RawSyntax) public init(_ lexeme: Lexer.Lexeme) { - if let kw = Keyword(lexeme.tokenText) { - self.init(kw) + if lexeme.rawTokenKind == .keyword { + self.init(Keyword(lexeme.tokenText)!) } else { self.init(nonKeyword: lexeme.rawTokenKind) } From f1430089338bc3171ac01748036beecd40e3b16f Mon Sep 17 00:00:00 2001 From: stevenwong Date: Fri, 3 Mar 2023 16:31:06 +0800 Subject: [PATCH 08/12] Fix a typo in comment --- Sources/SwiftSyntax/generated/Keyword.swift | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Sources/SwiftSyntax/generated/Keyword.swift b/Sources/SwiftSyntax/generated/Keyword.swift index ce799d53d9e..2f3cb2f980d 100644 --- a/Sources/SwiftSyntax/generated/Keyword.swift +++ b/Sources/SwiftSyntax/generated/Keyword.swift @@ -735,7 +735,7 @@ public enum Keyword: UInt8, Hashable { } } - /// Whether the token kind is switched from being an identifier to being an identifier to a keyword in the lexer. + /// Whether the token kind is switched from being an identifier to a keyword in the lexer. /// This is true for keywords that used to be considered non-contextual. public var isLexerClassified: Bool { switch self { From 52f7ace434760e63681f81166bb83689adbc2b4d Mon Sep 17 00:00:00 2001 From: stevenwong Date: Fri, 3 Mar 2023 16:46:21 +0800 Subject: [PATCH 09/12] Remove unused code --- Sources/SwiftParser/TokenPrecedence.swift | 2 -- 1 file changed, 2 deletions(-) diff --git a/Sources/SwiftParser/TokenPrecedence.swift b/Sources/SwiftParser/TokenPrecedence.swift index 788ec1ef6f7..9701b62cd00 100644 --- a/Sources/SwiftParser/TokenPrecedence.swift +++ b/Sources/SwiftParser/TokenPrecedence.swift @@ -217,8 +217,6 @@ public enum TokenPrecedence: Comparable { .fileprivate, .internal, .private, .public, .static, // Functions .deinit, .func, .`init`, .subscript, - // Macro - .macro, // Variables .let, .var, // Operator stuff From 6057c0baa624e49a92f1dbff80277861bea26070 Mon Sep 17 00:00:00 2001 From: stevenwong Date: Fri, 3 Mar 2023 16:47:42 +0800 Subject: [PATCH 10/12] Add args in the new recovery function callings --- Sources/SwiftParser/Declarations.swift | 3 ++- Sources/SwiftParser/TopLevel.swift | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/Sources/SwiftParser/Declarations.swift b/Sources/SwiftParser/Declarations.swift index be9e417ddc7..b3368e3c3de 100644 --- a/Sources/SwiftParser/Declarations.swift +++ b/Sources/SwiftParser/Declarations.swift @@ -1375,7 +1375,8 @@ extension Parser { var effectSpecifiers = self.parseDeclEffectSpecifiers() let output: RawReturnClauseSyntax? - if let _ = self.canRecoverTo(TokenSpec(.arrow)) { + + if self.at(.arrow) || self.canRecoverTo(TokenSpec(.arrow, recoveryPrecedence: .weakPunctuator, allowAtStartOfLine: false)) != nil { output = self.parseFunctionReturnClause(effectSpecifiers: &effectSpecifiers, allowNamedOpaqueResultType: true) } else { output = nil diff --git a/Sources/SwiftParser/TopLevel.swift b/Sources/SwiftParser/TopLevel.swift index ea85f31fb15..73227a0065e 100644 --- a/Sources/SwiftParser/TopLevel.swift +++ b/Sources/SwiftParser/TopLevel.swift @@ -98,7 +98,7 @@ extension Parser { /// This function is used when parsing places where function bodies are /// optional - like the function requirements in protocol declarations. mutating func parseOptionalCodeBlock(allowInitDecl: Bool = true) -> RawCodeBlockSyntax? { - guard let _ = self.canRecoverTo(TokenSpec(.leftBrace)) else { + guard self.at(.leftBrace) || self.canRecoverTo(TokenSpec(.leftBrace, allowAtStartOfLine: false)) != nil else { return nil } return self.parseCodeBlock(allowInitDecl: allowInitDecl) From 026d472c4722e097030f3229864474679896746a Mon Sep 17 00:00:00 2001 From: stevenwong Date: Wed, 8 Mar 2023 10:39:05 +0800 Subject: [PATCH 11/12] Let recovery respect to allowAtStartOfLine --- Sources/SwiftParser/Declarations.swift | 2 +- Sources/SwiftParser/Recovery.swift | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/Sources/SwiftParser/Declarations.swift b/Sources/SwiftParser/Declarations.swift index b3368e3c3de..0d56ef0964a 100644 --- a/Sources/SwiftParser/Declarations.swift +++ b/Sources/SwiftParser/Declarations.swift @@ -1376,7 +1376,7 @@ extension Parser { let output: RawReturnClauseSyntax? - if self.at(.arrow) || self.canRecoverTo(TokenSpec(.arrow, recoveryPrecedence: .weakPunctuator, allowAtStartOfLine: false)) != nil { + if self.at(.arrow) || self.canRecoverTo(TokenSpec(.arrow, allowAtStartOfLine: false)) != nil { output = self.parseFunctionReturnClause(effectSpecifiers: &effectSpecifiers, allowNamedOpaqueResultType: true) } else { output = nil diff --git a/Sources/SwiftParser/Recovery.swift b/Sources/SwiftParser/Recovery.swift index 21b4f7f7c9e..4e48f45de8c 100644 --- a/Sources/SwiftParser/Recovery.swift +++ b/Sources/SwiftParser/Recovery.swift @@ -70,9 +70,10 @@ extension Parser.Lookahead { let initialTokensConsumed = self.tokensConsumed let recoveryPrecedence = min(spec1.recoveryPrecedence, spec2.recoveryPrecedence, spec3.recoveryPrecedence) + let shouldSkipOverNewlines = recoveryPrecedence.shouldSkipOverNewlines && spec1.allowAtStartOfLine && spec2.allowAtStartOfLine && spec3.allowAtStartOfLine while !self.at(.eof) { - if !recoveryPrecedence.shouldSkipOverNewlines, self.currentToken.isAtStartOfLine { + if !shouldSkipOverNewlines, self.currentToken.isAtStartOfLine { break } let matchedSpec: TokenSpec? From 396ca71d85602d8b30d9cf2ac0c2aa4168f07601 Mon Sep 17 00:00:00 2001 From: stevenwong Date: Wed, 8 Mar 2023 10:40:58 +0800 Subject: [PATCH 12/12] Update a testcase for the change --- Tests/SwiftParserTest/DeclarationTests.swift | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Tests/SwiftParserTest/DeclarationTests.swift b/Tests/SwiftParserTest/DeclarationTests.swift index 78b60525a6e..628ee4d159c 100644 --- a/Tests/SwiftParserTest/DeclarationTests.swift +++ b/Tests/SwiftParserTest/DeclarationTests.swift @@ -917,7 +917,7 @@ final class DeclarationTests: XCTestCase { diagnostics: [ DiagnosticSpec(locationMarker: "1️⃣", message: "expected ':' in parameter"), DiagnosticSpec(locationMarker: "2️⃣", message: "expected ')' to end parameter clause"), - DiagnosticSpec(locationMarker: "3️⃣", message: "unexpected code ': Int)' in function"), + DiagnosticSpec(locationMarker: "3️⃣", message: "extraneous code ': Int) {}' at top level"), ] ) }