From 43b8ef7289abb4da6aa980c8dc18c172ab9da998 Mon Sep 17 00:00:00 2001 From: Alex Hoppen Date: Wed, 29 Mar 2023 00:28:19 +0100 Subject: [PATCH 01/12] Import regex parsing tests from test/StringProcessing/Parse --- .../ForwardSlashRegexDisabledTests.swift | 94 + ...orwardSlashRegexSkippingAllowedTests.swift | 158 ++ ...orwardSlashRegexSkippingInvalidTests.swift | 331 +++ .../ForwardSlashRegexSkippingTests.swift | 607 +++++ .../translated/ForwardSlashRegexTests.swift | 2069 +++++++++++++++++ .../translated/PrefixSlashTests.swift | 77 + .../RegexParseEndOfBufferTests.swift | 21 + .../translated/RegexParseErrorTests.swift | 251 ++ .../translated/RegexTests.swift | 72 + 9 files changed, 3680 insertions(+) create mode 100644 Tests/SwiftParserTest/translated/ForwardSlashRegexDisabledTests.swift create mode 100644 Tests/SwiftParserTest/translated/ForwardSlashRegexSkippingAllowedTests.swift create mode 100644 Tests/SwiftParserTest/translated/ForwardSlashRegexSkippingInvalidTests.swift create mode 100644 Tests/SwiftParserTest/translated/ForwardSlashRegexSkippingTests.swift create mode 100644 Tests/SwiftParserTest/translated/ForwardSlashRegexTests.swift create mode 100644 Tests/SwiftParserTest/translated/PrefixSlashTests.swift create mode 100644 Tests/SwiftParserTest/translated/RegexParseEndOfBufferTests.swift create mode 100644 Tests/SwiftParserTest/translated/RegexParseErrorTests.swift create mode 100644 Tests/SwiftParserTest/translated/RegexTests.swift diff --git a/Tests/SwiftParserTest/translated/ForwardSlashRegexDisabledTests.swift b/Tests/SwiftParserTest/translated/ForwardSlashRegexDisabledTests.swift new file mode 100644 index 00000000000..655ef15b181 --- /dev/null +++ b/Tests/SwiftParserTest/translated/ForwardSlashRegexDisabledTests.swift @@ -0,0 +1,94 @@ +// This test file has been translated from swift/test/StringProcessing/Parse/forward-slash-regex-disabled.swift + +import XCTest + +final class ForwardSlashRegexDisabledTests: XCTestCase { + func testForwardSlashRegexDisabled1() { + AssertParse( + """ + prefix operator / + prefix operator ^/ + prefix operator /^/ + """ + ) + } + + func testForwardSlashRegexDisabled2() { + AssertParse( + """ + precedencegroup P { + associativity: left + } + """ + ) + } + + func testForwardSlashRegexDisabled3() { + AssertParse( + """ + // The divisions in the body of the below operators make sure we don't try and + // consider them to be ending delimiters of a regex. + infix operator /^/ : P + func /^/ (lhs: Int, rhs: Int) -> Int { 1 / 2 } + """ + ) + } + + func testForwardSlashRegexDisabled4() { + AssertParse( + """ + infix operator /^ : P + func /^ (lhs: Int, rhs: Int) -> Int { 1 / 2 } + """ + ) + } + + func testForwardSlashRegexDisabled5() { + AssertParse( + """ + infix operator ^^/ : P + func ^^/ (lhs: Int, rhs: Int) -> Int { 1 / 2 } + """ + ) + } + + func testForwardSlashRegexDisabled6() { + AssertParse( + """ + _ = #/x/# + """ + ) + } + + func testForwardSlashRegexDisabled7() { + AssertParse( + """ + _ = /x/ + """, + diagnostics: [ + // TODO: Old parser expected error on line 1: '/' is not a prefix unary operator + // TODO: Old parser expected error on line 1: cannot find 'x' in scope + // TODO: Old parser expected error on line 1: '/' is not a postfix unary operator + ] + ) + } + + func testForwardSlashRegexDisabled8() { + AssertParse( + """ + func baz(_ x: (Int, Int) -> Int, _ y: (Int, Int) -> Int) {} + """ + ) + } + + func testForwardSlashRegexDisabled9() { + AssertParse( + """ + baz(/, /) + baz(/^, /) + baz(^^/, /) + """ + ) + } + +} diff --git a/Tests/SwiftParserTest/translated/ForwardSlashRegexSkippingAllowedTests.swift b/Tests/SwiftParserTest/translated/ForwardSlashRegexSkippingAllowedTests.swift new file mode 100644 index 00000000000..a743505b0d3 --- /dev/null +++ b/Tests/SwiftParserTest/translated/ForwardSlashRegexSkippingAllowedTests.swift @@ -0,0 +1,158 @@ +// This test file has been translated from swift/test/StringProcessing/Parse/forward-slash-regex-skipping-allowed.swift + +import XCTest + +final class ForwardSlashRegexSkippingAllowedTests: XCTestCase { + func testForwardSlashRegexSkippingAllowed1() { + AssertParse( + """ + // Make sure we can skip in all of the below cases. + """ + ) + } + + func testForwardSlashRegexSkippingAllowed2() { + AssertParse( + #""" + // The printing implementation differs in asserts and no-asserts builds, it will + // either print `"Parse.NumFunctionsParsed" 0` or not print it at all. Make sure + // we don't output any non-zero value. + // CHECK-NOT: {{"Parse.NumFunctionsParsed" [^0]}} + """# + ) + } + + func testForwardSlashRegexSkippingAllowed3() { + AssertParse( + """ + // Ensures there is a parse error + var 1️⃣: Int + """, + diagnostics: [ + // TODO: Old parser expected error on line 2: expected pattern + DiagnosticSpec(message: "expected pattern in variable"), + ] + ) + } + + func testForwardSlashRegexSkippingAllowed4() { + AssertParse( + """ + // Balanced `{}`, so okay. + func a() { 1️⃣/ {}/ } + """, + diagnostics: [ + DiagnosticSpec(message: "unexpected code '/ {}/' in function"), + ] + ) + } + + func testForwardSlashRegexSkippingAllowed5() { + AssertParse( + #""" + func b() { 1️⃣/ \{}/ } + """#, + diagnostics: [ + DiagnosticSpec(message: #"unexpected code '/ \{}/' in function"#), + ] + ) + } + + func testForwardSlashRegexSkippingAllowed6() { + AssertParse( + #""" + func c() { 1️⃣/ {"{"}/ } + """#, + diagnostics: [ + DiagnosticSpec(message: #"unexpected code '/ {"{"}/' in function"#), + ] + ) + } + + func testForwardSlashRegexSkippingAllowed7() { + AssertParse( + """ + // Some cases of infix '/' that we should continue to skip. + func d() { + _ = 1 / 2 + 3 * 4 + _ = 1 / 2 / 3 / 4 + } + """ + ) + } + + func testForwardSlashRegexSkippingAllowed8() { + AssertParse( + #""" + func e() { + let arr = [1, 2, 3] + _ = arr.reduce(0, /) / 2 + func foo(_ i: Int, _ fn: () -> Void) {} + foo(1 / 2 / 3, { print("}}}{{{") }) + } + """# + ) + } + + func testForwardSlashRegexSkippingAllowed9() { + AssertParse( + """ + // Some cases of prefix '/' that we should continue to skip. + prefix operator / + prefix func / (_ x: T) -> T { x } + """ + ) + } + + func testForwardSlashRegexSkippingAllowed10() { + AssertParse( + """ + enum E { + case e + func foo(_ x: T) {} + } + """ + ) + } + + func testForwardSlashRegexSkippingAllowed11() { + AssertParse( + """ + func f() { + _ = /E.e + (/E.e).foo(/0) + func foo(_ x: T, _ y: U) {} + foo(/E.e, /E.e) + foo((/E.e), /E.e) + foo((/)(E.e), /E.e) + func bar(_ x: T) -> Int { 0 } + _ = bar(/E.e) / 2 + } + """ + ) + } + + func testForwardSlashRegexSkippingAllowed12() { + AssertParse( + """ + postfix operator / + prefix func / (_ x: T) -> T { x } + """ + ) + } + + func testForwardSlashRegexSkippingAllowed13() { + AssertParse( + """ + // Some cases of postfix '/' that we should continue to skip. + func g() { + _ = 0/ + _ = 0/ / 1/ + _ = 1/ + 1/ + _ = 1 + 2/ + } + """ + ) + } + +} diff --git a/Tests/SwiftParserTest/translated/ForwardSlashRegexSkippingInvalidTests.swift b/Tests/SwiftParserTest/translated/ForwardSlashRegexSkippingInvalidTests.swift new file mode 100644 index 00000000000..9136dfdd3df --- /dev/null +++ b/Tests/SwiftParserTest/translated/ForwardSlashRegexSkippingInvalidTests.swift @@ -0,0 +1,331 @@ +// This test file has been translated from swift/test/StringProcessing/Parse/forward-slash-regex-skipping-invalid.swift + +import XCTest + +final class ForwardSlashRegexSkippingInvalidTests: XCTestCase { + func testForwardSlashRegexSkippingInvalid1() { + AssertParse( + """ + // We don't consider this a regex literal when skipping as it has an initial + // space. + func a() { _ = 1️⃣/ x*/ } + """, + diagnostics: [ + // TODO: Old parser expected error on line 3: unexpected end of block comment + DiagnosticSpec(message: "expected expression in function"), + DiagnosticSpec(message: "unexpected code '/ x*/' in function"), + ] + ) + } + + func testForwardSlashRegexSkippingInvalid2() { + AssertParse( + """ + // Same because of unbalanced ')' + func b() { _ = /x1️⃣)*/ } + """, + diagnostics: [ + // TODO: Old parser expected error on line 2: unexpected end of block comment + DiagnosticSpec(message: "unexpected code ')*/' in function"), + ] + ) + } + + func testForwardSlashRegexSkippingInvalid3() { + AssertParse( + """ + // These also fail the heuristic, but have unbalanced `{` `}`, so we don't skip. + func c() { _ = 1️⃣/ x}2️⃣*/ } + func d() { _ = 3️⃣/ x{*/ } + """, + diagnostics: [ + // TODO: Old parser expected error on line 2: regex literal may not start with space; add backslash to escape + DiagnosticSpec(locationMarker: "1️⃣", message: "expected expression in function"), + DiagnosticSpec(locationMarker: "1️⃣", message: "unexpected code '/ x' in function"), + DiagnosticSpec(locationMarker: "2️⃣", message: "unexpected code '*/ }' before function"), + // TODO: Old parser expected error on line 3: regex literal may not start with space; add backslash to escape + DiagnosticSpec(locationMarker: "3️⃣", message: "expected expression in function"), + DiagnosticSpec(locationMarker: "3️⃣", message: "expected '}' to end function"), + DiagnosticSpec(locationMarker: "3️⃣", message: "extraneous code '/ x{*/ }' at top level"), + ] + ) + } + + func testForwardSlashRegexSkippingInvalid4() { + AssertParse( + """ + // Unterminated, and unbalanced `{}`. + func e() { + _ = 1️⃣/ } + """, + diagnostics: [ + DiagnosticSpec(message: "expected expression in function"), + DiagnosticSpec(message: "unexpected code '/' in function"), + ] + ) + } + + func testForwardSlashRegexSkippingInvalid5() { + AssertParse( + """ + 1️⃣} + func f() { + _ = 2️⃣/ { + } + """, + diagnostics: [ + // TODO: Old parser expected error on line 0: unterminated regex literal + // TODO: Old parser expected error on line 0: regex literal may not start with space; add backslash to escape + DiagnosticSpec(locationMarker: "1️⃣", message: "unexpected brace before function"), + // TODO: Old parser expected error on line 3: unterminated regex literal + // TODO: Old parser expected error on line 3: regex literal may not start with space; add backslash to escape + DiagnosticSpec(locationMarker: "2️⃣", message: "expected expression in function"), + DiagnosticSpec(locationMarker: "2️⃣", message: "expected '}' to end function"), + DiagnosticSpec(locationMarker: "2️⃣", message: "extraneous code at top level"), + ] + ) + } + + func testForwardSlashRegexSkippingInvalid6() { + AssertParse( + """ + func g() { + _ = /x } + """ + ) + } + + func testForwardSlashRegexSkippingInvalid7() { + AssertParse( + """ + 1️⃣} + func h() { + _ = /x { + } 2️⃣// The above cannot a regex literal so we skip; this `}` is to balance things out. + """, + diagnostics: [ + // TODO: Old parser expected error on line 1: extraneous '}' at top level + DiagnosticSpec(locationMarker: "1️⃣", message: "unexpected brace before function"), + DiagnosticSpec(locationMarker: "2️⃣", message: "expected '}' to end function"), + ] + ) + } + + func testForwardSlashRegexSkippingInvalid8() { + AssertParse( + #""" + 1️⃣} + func i() { + _ = /x 2️⃣"[abc] { + } + """#, + diagnostics: [ + DiagnosticSpec(locationMarker: "1️⃣", message: "unexpected brace before function"), + // TODO: Old parser expected error on line 3: unterminated string literal + DiagnosticSpec(locationMarker: "2️⃣", message: #"unexpected code '"[abc] {' in function"#), + ] + ) + } + + func testForwardSlashRegexSkippingInvalid9() { + AssertParse( + """ + func j() { + _ = 1️⃣/^ [abc] { + } + """, + diagnostics: [ + // TODO: Old parser expected error on line 2: unterminated regex literal + DiagnosticSpec(message: "expected expression in function"), + DiagnosticSpec(message: "expected '}' to end function"), + DiagnosticSpec(message: "extraneous code at top level"), + ] + ) + } + + func testForwardSlashRegexSkippingInvalid10() { + AssertParse( + #""" + func k() { + _ = 1️⃣/^ "[abc] { + } + """#, + diagnostics: [ + // TODO: Old parser expected error on line 2: unterminated string literal + DiagnosticSpec(message: "expected expression in function"), + DiagnosticSpec(message: #"unexpected code '/^ "[abc] {' in function"#), + ] + ) + } + + func testForwardSlashRegexSkippingInvalid11() { + AssertParse( + """ + func l() { + _ = 1️⃣/^ } abc { + } + """, + diagnostics: [ + // TODO: Old parser expected error on line 2: unterminated regex literal + DiagnosticSpec(message: "expected expression in function"), + DiagnosticSpec(message: "unexpected code '/^' in function"), + ] + ) + } + + func testForwardSlashRegexSkippingInvalid12() { + AssertParse( + #""" + func m() { + _ = 1️⃣/ " + } + """#, + diagnostics: [ + // TODO: Old parser expected error on line 2: unterminated string literal + DiagnosticSpec(message: "expected expression in function"), + DiagnosticSpec(message: #"unexpected code '/ "' in function"#), + ] + ) + } + + func testForwardSlashRegexSkippingInvalid13() { + AssertParse( + #""" + 1️⃣} + // Unbalanced `}`, make sure we don't consider the string literal `{`. + func n() { 2️⃣/ "{"}3️⃣/ } + """#, + diagnostics: [ + // TODO: Old parser expected error on line 1: extraneous '}' at top level + DiagnosticSpec(locationMarker: "1️⃣", message: "unexpected brace before function"), + // TODO: Old parser expected error on line 3: regex literal may not start with space; add backslash to escape + DiagnosticSpec(locationMarker: "2️⃣", message: #"unexpected code '/ "{"' in function"#), + DiagnosticSpec(locationMarker: "3️⃣", message: "extraneous code '/ }' at top level"), + ] + ) + } + + func testForwardSlashRegexSkippingInvalid14() { + AssertParse( + """ + func o() { + _ = { + 0 + /x}}1️⃣} / + 2 + } + } + """, + diagnostics: [ + // TODO: Old parser expected error on line 4: extraneous '}' at top level + // TODO: Old parser expected error on line 4: consecutive statements on a line must be separated by ';' + // TODO: Old parser expected error on line 4: unterminated regex literal + // TODO: Old parser expected warning on line 4: regular expression literal is unused + DiagnosticSpec(message: "extraneous code at top level"), + // TODO: Old parser expected warning on line 5: integer literal is unused + // TODO: Old parser expected error on line 6: extraneous '}' at top level + // TODO: Old parser expected error on line 7: extraneous '}' at top level + ] + ) + } + + func testForwardSlashRegexSkippingInvalid15() { + AssertParse( + """ + func p() { + _ = 2 + /x} 1️⃣/ + """, + diagnostics: [ + DiagnosticSpec(message: "extraneous code '/' at top level"), + ] + ) + } + + func testForwardSlashRegexSkippingInvalid16() { + AssertParse( + """ + .bitWidth + 1️⃣} + """, + diagnostics: [ + // TODO: Old parser expected error on line 0: consecutive statements on a line must be separated by ';' + // TODO: Old parser expected error on line 0: unterminated regex literal + // TODO: Old parser expected error on line 1: value of type 'Regex' has no member 'bitWidth' + // TODO: Old parser expected error on line 2: extraneous '}' at top level + DiagnosticSpec(message: "extraneous brace at top level"), + ] + ) + } + + func testForwardSlashRegexSkippingInvalid17() { + AssertParse( + """ + func err1() { _ = 1️⃣/ 0xG}2️⃣/ } + """, + diagnostics: [ + // TODO: Old parser expected error on line 1: regex literal may not start with space; add backslash to escape + DiagnosticSpec(locationMarker: "1️⃣", message: "expected expression in function"), + DiagnosticSpec(locationMarker: "1️⃣", message: "unexpected code '/ 0xG' in function"), + DiagnosticSpec(locationMarker: "2️⃣", message: "extraneous code '/ }' at top level"), + ] + ) + } + + func testForwardSlashRegexSkippingInvalid18() { + AssertParse( + """ + func err2() { _ = 1️⃣/ 0oG}2️⃣/ } + """, + diagnostics: [ + // TODO: Old parser expected error on line 1: regex literal may not start with space; add backslash to escape + DiagnosticSpec(locationMarker: "1️⃣", message: "expected expression in function"), + DiagnosticSpec(locationMarker: "1️⃣", message: "unexpected code '/ 0oG' in function"), + DiagnosticSpec(locationMarker: "2️⃣", message: "extraneous code '/ }' at top level"), + ] + ) + } + + func testForwardSlashRegexSkippingInvalid19() { + AssertParse( + #""" + func err3() { _ = 1️⃣/ {"/ } + """#, + diagnostics: [ + // TODO: Old parser expected error on line 1: regex literal may not start with space; add backslash to escape + DiagnosticSpec(message: "expected expression in function"), + DiagnosticSpec(message: "expected '}' to end function"), + DiagnosticSpec(message: #"extraneous code '/ {"/ }' at top level"#), + ] + ) + } + + func testForwardSlashRegexSkippingInvalid20() { + AssertParse( + """ + func err4() { _ = 1️⃣/ {'/ } + """, + diagnostics: [ + // TODO: Old parser expected error on line 1: regex literal may not start with space; add backslash to escape + DiagnosticSpec(message: "expected expression in function"), + DiagnosticSpec(message: "expected '}' to end function"), + DiagnosticSpec(message: "extraneous code '/ {'/ }' at top level"), + ] + ) + } + + func testForwardSlashRegexSkippingInvalid21() { + AssertParse( + """ + func err5() { _ = 1️⃣/ {<#placeholder#>/ } + """, + diagnostics: [ + // TODO: Old parser expected error on line 1: regex literal may not start with space; add backslash to escape + DiagnosticSpec(message: "expected expression in function"), + DiagnosticSpec(message: "expected '}' to end function"), + DiagnosticSpec(message: "extraneous code '/ {<#placeholder#>/ }' at top level"), + ] + ) + } + +} diff --git a/Tests/SwiftParserTest/translated/ForwardSlashRegexSkippingTests.swift b/Tests/SwiftParserTest/translated/ForwardSlashRegexSkippingTests.swift new file mode 100644 index 00000000000..a5294a20a55 --- /dev/null +++ b/Tests/SwiftParserTest/translated/ForwardSlashRegexSkippingTests.swift @@ -0,0 +1,607 @@ +// This test file has been translated from swift/test/StringProcessing/Parse/forward-slash-regex-skipping.swift + +import XCTest + +final class ForwardSlashRegexSkippingTests: XCTestCase { + func testForwardSlashRegexSkipping1() { + AssertParse( + """ + // Make sure we properly handle `/.../` regex literals in skipped function + // bodies. Currently we detect them and avoid skipping, but in the future we + // ought to be able to skip over them. + """ + ) + } + + func testForwardSlashRegexSkipping2() { + AssertParse( + """ + prefix operator ^^ + prefix func ^^ (_ x: T) -> T { x } + """ + ) + } + + func testForwardSlashRegexSkipping3() { + AssertParse( + #""" + struct A { + static let r = /test":"(.*?)"/ + } + """# + ) + } + + func testForwardSlashRegexSkipping4() { + AssertParse( + """ + struct B { + static let r = /x*/ + } + """ + ) + } + + func testForwardSlashRegexSkipping5() { + AssertParse( + """ + struct C { + func foo() { + let r = /x*/ + } + } + """ + ) + } + + func testForwardSlashRegexSkipping6() { + AssertParse( + """ + struct D { + func foo() { + func bar() { + let r = /x}}*/ + }1️⃣ + """, + diagnostics: [ + DiagnosticSpec(message: "expected '}' to end function"), + DiagnosticSpec(message: "expected '}' to end struct"), + ] + ) + } + + func testForwardSlashRegexSkipping7() { + AssertParse( + """ + 1️⃣} + } + func a() { _ = /abc}}*/ } + """, + diagnostics: [ + DiagnosticSpec(message: "unexpected braces before function"), + ] + ) + } + + func testForwardSlashRegexSkipping8() { + AssertParse( + #""" + func b() { _ = /\// } + """# + ) + } + + func testForwardSlashRegexSkipping9() { + AssertParse( + #""" + func c() { _ = /\\/ } + """# + ) + } + + func testForwardSlashRegexSkipping10() { + AssertParse( + """ + func d() { _ = ^^/x}1️⃣}*/ } + """, + diagnostics: [ + DiagnosticSpec(message: "extraneous code '}*/ }' at top level"), + ] + ) + } + + func testForwardSlashRegexSkipping11() { + AssertParse( + """ + func e() { _ = (^^/x1️⃣}2️⃣}*/) } + """, + diagnostics: [ + DiagnosticSpec(locationMarker: "1️⃣", message: "expected ')' to end tuple"), + DiagnosticSpec(locationMarker: "2️⃣", message: "extraneous code '}*/) }' at top level"), + ] + ) + } + + func testForwardSlashRegexSkipping12() { + AssertParse( + """ + func f() { _ = ^^/^x}1️⃣}*/ } + """, + diagnostics: [ + DiagnosticSpec(message: "extraneous code '}*/ }' at top level"), + ] + ) + } + + func testForwardSlashRegexSkipping13() { + AssertParse( + #""" + func g() { _ = "\(/x}}*/)" } + """# + ) + } + + func testForwardSlashRegexSkipping14() { + AssertParse( + #""" + func h() { _ = "\(^^/x1️⃣}}*/)" } + """#, + diagnostics: [ + DiagnosticSpec(message: "unexpected code '}}*/' in string literal"), + ] + ) + } + + func testForwardSlashRegexSkipping15() { + AssertParse( + #""" + func i() { + func foo(_ x: T, y: T) {} + foo(/}}*/, y: /"/) + } + """# + ) + } + + func testForwardSlashRegexSkipping16() { + AssertParse( + """ + func j() { + _ = { + 0 + /x}}}/ + 2 + } + } + """ + ) + } + + func testForwardSlashRegexSkipping17() { + AssertParse( + """ + func k() { + _ = 2 + / 1 / .bitWidth + } + """ + ) + } + + func testForwardSlashRegexSkipping18() { + AssertParse( + """ + func l() { + _ = 2 + /x}*/ .self1️⃣ + """, + diagnostics: [ + DiagnosticSpec(message: "expected '}' to end function"), + ] + ) + } + + func testForwardSlashRegexSkipping19() { + AssertParse( + """ + 1️⃣} + """, + diagnostics: [ + DiagnosticSpec(message: "extraneous brace at top level"), + ] + ) + } + + func testForwardSlashRegexSkipping20() { + AssertParse( + """ + func m() { + _ = 2 + / 1 / + .bitWidth + } + """ + ) + } + + func testForwardSlashRegexSkipping21() { + AssertParse( + """ + func n() { + _ = 2 + /x}/1️⃣ + """, + diagnostics: [ + DiagnosticSpec(message: "expected '}' to end function"), + ] + ) + } + + func testForwardSlashRegexSkipping22() { + AssertParse( + """ + .bitWidth + 1️⃣} + """, + diagnostics: [ + DiagnosticSpec(message: "extraneous brace at top level"), + ] + ) + } + + func testForwardSlashRegexSkipping23() { + AssertParse( + """ + func o() { + _ = /x// comment + } + """ + ) + } + + func testForwardSlashRegexSkipping24() { + AssertParse( + """ + func p() { + _ = /x // comment + } + """ + ) + } + + func testForwardSlashRegexSkipping25() { + AssertParse( + """ + func q() { + _ = /x/*comment*/ + } + """ + ) + } + + func testForwardSlashRegexSkipping26() { + AssertParse( + """ + func r() { _ = /[(0)]/ } + """ + ) + } + + func testForwardSlashRegexSkipping27() { + AssertParse( + """ + func s() { _ = /(x)/ } + """ + ) + } + + func testForwardSlashRegexSkipping28() { + AssertParse( + """ + func t() { _ = /[1️⃣)]/ } + """, + diagnostics: [ + DiagnosticSpec(message: "expected ']' to end array"), + DiagnosticSpec(message: "unexpected code ')]/' in function"), + ] + ) + } + + func testForwardSlashRegexSkipping29() { + AssertParse( + #""" + func u() { _ = /[a\1️⃣]2️⃣)]/ } + """#, + diagnostics: [ + DiagnosticSpec(locationMarker: "1️⃣", message: "expected root in key path"), + DiagnosticSpec(locationMarker: "2️⃣", message: "unexpected code ')]/' in function"), + ] + ) + } + + func testForwardSlashRegexSkipping30() { + AssertParse( + """ + func v() { _ = /([1️⃣)2️⃣])/ } + """, + diagnostics: [ + DiagnosticSpec(locationMarker: "1️⃣", message: "expected ']' to end array"), + DiagnosticSpec(locationMarker: "2️⃣", message: "unexpected code '])/' in function"), + ] + ) + } + + func testForwardSlashRegexSkipping31() { + AssertParse( + """ + func w() { _ = 1️⃣/]]][)]/ } + """, + diagnostics: [ + DiagnosticSpec(message: "expected expression in function"), + DiagnosticSpec(message: "expected '}' to end function"), + DiagnosticSpec(message: "extraneous code '/]]][)]/ }' at top level"), + ] + ) + } + + func testForwardSlashRegexSkipping32() { + AssertParse( + """ + func x() { _ = /,/ } + """ + ) + } + + func testForwardSlashRegexSkipping33() { + AssertParse( + """ + func y() { _ = /}/ } + """ + ) + } + + func testForwardSlashRegexSkipping34() { + AssertParse( + """ + func z() { _ = /]/ } + """ + ) + } + + func testForwardSlashRegexSkipping35() { + AssertParse( + """ + func a1() { _ = /:/ } + """ + ) + } + + func testForwardSlashRegexSkipping36() { + AssertParse( + """ + func a2() { _ = /;/ } + """ + ) + } + + func testForwardSlashRegexSkipping37() { + AssertParse( + """ + func a3() { _ = 1️⃣/)/ } + """, + diagnostics: [ + DiagnosticSpec(message: "expected expression in function"), + DiagnosticSpec(message: "unexpected code '/)/' in function"), + ] + ) + } + + func testForwardSlashRegexSkipping38() { + AssertParse( + """ + func a4() { _ = 1️⃣/ / } + """, + diagnostics: [ + // TODO: Old parser expected error on line 1: regex literal may not start with space; add backslash to escape + DiagnosticSpec(message: "expected expression in function"), + DiagnosticSpec(message: "unexpected code '/ /' in function"), + ] + ) + } + + func testForwardSlashRegexSkipping39() { + AssertParse( + #""" + func a5() { _ = /\ 1️⃣/ 2️⃣} + """#, + diagnostics: [ + DiagnosticSpec(locationMarker: "1️⃣", message: "expected root in key path"), + DiagnosticSpec(locationMarker: "2️⃣", message: "expected expression in function"), + ] + ) + } + + func testForwardSlashRegexSkipping40() { + AssertParse( + """ + prefix operator / + prefix func / (_ x: T) -> T { x } + """ + ) + } + + func testForwardSlashRegexSkipping41() { + AssertParse( + """ + enum E { + case e + func foo(_ x: T) {} + } + """ + ) + } + + func testForwardSlashRegexSkipping42() { + AssertParse( + #""" + func a7() { _ = /\/}/ } + """# + ) + } + + func testForwardSlashRegexSkipping43() { + AssertParse( + """ + // Make sure we don't emit errors for these. + func err1() { _ = /0xG/ } + """ + ) + } + + func testForwardSlashRegexSkipping44() { + AssertParse( + """ + func err2() { _ = /0oG/ } + """ + ) + } + + func testForwardSlashRegexSkipping45() { + AssertParse( + #""" + func err3() { _ = /"/ } + """# + ) + } + + func testForwardSlashRegexSkipping46() { + AssertParse( + """ + func err4() { _ = /'/ } + """ + ) + } + + func testForwardSlashRegexSkipping47() { + AssertParse( + """ + func err5() { _ = /<#placeholder#>/ } + """ + ) + } + + func testForwardSlashRegexSkipping48() { + AssertParse( + """ + func err6() { _ = ^^/1️⃣0xG/ } + """, + diagnostics: [ + DiagnosticSpec(message: "expected expression in prefix operator expression"), + DiagnosticSpec(message: "unexpected code '0xG/' in function"), + ] + ) + } + + func testForwardSlashRegexSkipping49() { + AssertParse( + """ + func err7() { _ = ^^/1️⃣0oG/ } + """, + diagnostics: [ + DiagnosticSpec(message: "expected expression in prefix operator expression"), + DiagnosticSpec(message: "unexpected code '0oG/' in function"), + ] + ) + } + + func testForwardSlashRegexSkipping50() { + AssertParse( + #""" + func err8() { _ = ^^/"/ }1️⃣ + """#, + diagnostics: [ + DiagnosticSpec(message: #"expected '"' to end string literal"#), + DiagnosticSpec(message: "expected '}' to end function"), + ] + ) + } + + func testForwardSlashRegexSkipping51() { + AssertParse( + """ + func err9() { _ = ^^/'/ }1️⃣ + """, + diagnostics: [ + DiagnosticSpec(message: "expected ''' in string literal"), + DiagnosticSpec(message: "expected '}' to end function"), + ] + ) + } + + func testForwardSlashRegexSkipping52() { + AssertParse( + """ + func err10() { _ = ^^/<#placeholder#>/ } + """ + ) + } + + func testForwardSlashRegexSkipping53() { + AssertParse( + """ + func err11() { _ = (^^/1️⃣0xG/) } + """, + diagnostics: [ + DiagnosticSpec(message: "expected expression in prefix operator expression"), + DiagnosticSpec(message: "unexpected code '0xG/' in tuple"), + ] + ) + } + + func testForwardSlashRegexSkipping54() { + AssertParse( + """ + func err12() { _ = (^^/1️⃣0oG/) } + """, + diagnostics: [ + DiagnosticSpec(message: "expected expression in prefix operator expression"), + DiagnosticSpec(message: "unexpected code '0oG/' in tuple"), + ] + ) + } + + func testForwardSlashRegexSkipping55() { + AssertParse( + #""" + func err13() { _ = (^^/"/) }1️⃣ + """#, + diagnostics: [ + DiagnosticSpec(message: #"expected '"' to end string literal"#), + DiagnosticSpec(message: "expected ')' to end tuple"), + DiagnosticSpec(message: "expected '}' to end function"), + ] + ) + } + + func testForwardSlashRegexSkipping56() { + AssertParse( + """ + func err14() { _ = (^^/'/) }1️⃣ + """, + diagnostics: [ + DiagnosticSpec(message: "expected ''' in string literal"), + DiagnosticSpec(message: "expected ')' to end tuple"), + DiagnosticSpec(message: "expected '}' to end function"), + ] + ) + } + + func testForwardSlashRegexSkipping57() { + AssertParse( + """ + func err15() { _ = (^^/<#placeholder#>/) } + """ + ) + } + +} diff --git a/Tests/SwiftParserTest/translated/ForwardSlashRegexTests.swift b/Tests/SwiftParserTest/translated/ForwardSlashRegexTests.swift new file mode 100644 index 00000000000..69c9cdcd7d3 --- /dev/null +++ b/Tests/SwiftParserTest/translated/ForwardSlashRegexTests.swift @@ -0,0 +1,2069 @@ +// This test file has been translated from swift/test/StringProcessing/Parse/forward-slash-regex.swift + +import XCTest + +final class ForwardSlashRegexTests: XCTestCase { + func testForwardSlashRegex1() { + AssertParse( + """ + prefix operator / + prefix operator ^/ + prefix operator /^/ + """ + ) + } + + func testForwardSlashRegex2() { + AssertParse( + """ + prefix func ^/ (_ x: T) -> T { x } + """ + ) + } + + func testForwardSlashRegex3() { + AssertParse( + """ + prefix operator !! + """ + ) + } + + func testForwardSlashRegex4() { + AssertParse( + """ + prefix func !! (_ x: T) -> T { x } + """ + ) + } + + func testForwardSlashRegex5() { + AssertParse( + """ + prefix operator ^^ + """ + ) + } + + func testForwardSlashRegex6() { + AssertParse( + """ + prefix func ^^ (_ x: T) -> T { x } + """ + ) + } + + func testForwardSlashRegex7() { + AssertParse( + """ + precedencegroup P { + associativity: left + } + """ + ) + } + + func testForwardSlashRegex8() { + AssertParse( + """ + // The divisions in the body of the below operators make sure we don't try and + // consider them to be ending delimiters of a regex. + infix operator /^/ : P + func /^/ (lhs: Int, rhs: Int) -> Int { 1 / 2 } + """ + ) + } + + func testForwardSlashRegex9() { + AssertParse( + """ + infix operator /^ : P + func /^ (lhs: Int, rhs: Int) -> Int { 1 / 2 } + """ + ) + } + + func testForwardSlashRegex10() { + AssertParse( + """ + infix operator ^^/ : P + func ^^/ (lhs: Int, rhs: Int) -> Int { 1 / 2 } + """ + ) + } + + func testForwardSlashRegex11() { + AssertParse( + """ + let i = 01️⃣ /^/2️⃣ 1/^/3 + """, + diagnostics: [ + DiagnosticSpec(locationMarker: "1️⃣", message: "consecutive statements on a line must be separated by ';'"), + DiagnosticSpec(locationMarker: "2️⃣", message: "consecutive statements on a line must be separated by ';'"), + ] + ) + } + + func testForwardSlashRegex12() { + AssertParse( + """ + let x = /abc/ + """ + ) + } + + func testForwardSlashRegex13() { + AssertParse( + """ + _ = /abc/ + """ + ) + } + + func testForwardSlashRegex14() { + AssertParse( + """ + _ = /x/.self + """ + ) + } + + func testForwardSlashRegex15() { + AssertParse( + #""" + _ = /\// + """# + ) + } + + func testForwardSlashRegex16() { + AssertParse( + #""" + _ = /\\/ + """# + ) + } + + func testForwardSlashRegex17() { + AssertParse( + """ + // This is just here to appease typo correction. + let y = 0 + """ + ) + } + + func testForwardSlashRegex18() { + AssertParse( + """ + // These unfortunately become prefix `=` and infix `=/` respectively. We could + // likely improve the diagnostic though. + do { + let z1️⃣=/0/ + } + """, + diagnostics: [ + // TODO: Old parser expected error on line 4: type annotation missing in pattern + // TODO: Old parser expected error on line 4: consecutive statements on a line must be separated by ';' + // TODO: Old parser expected error on line 4: expected expression + DiagnosticSpec(message: "unexpected code '=/0/' in 'do' statement"), + ] + ) + } + + func testForwardSlashRegex19() { + AssertParse( + """ + do { + _=/0/ + } + """, + diagnostics: [ + // TODO: Old parser expected error on line 2: '_' can only appear in a pattern or on the left side of an assignment + // TODO: Old parser expected error on line 2: cannot find operator '=/' in scope + // TODO: Old parser expected error on line 2: '/' is not a postfix unary operator + ] + ) + } + + func testForwardSlashRegex20() { + AssertParse( + """ + // No closing '/' so a prefix operator. + """ + ) + } + + func testForwardSlashRegex21() { + AssertParse( + """ + _ = /x + """, + diagnostics: [ + // TODO: Old parser expected error on line 1: '/' is not a prefix unary operator + ] + ) + } + + func testForwardSlashRegex22() { + AssertParse( + """ + _ = !/x/ + """, + diagnostics: [ + // TODO: Old parser expected error on line 1: cannot convert value of type 'Regex' to expected argument type 'Bool' + ] + ) + } + + func testForwardSlashRegex23() { + AssertParse( + """ + _ = (!/x/) + """, + diagnostics: [ + // TODO: Old parser expected error on line 1: cannot convert value of type 'Regex' to expected argument type 'Bool' + ] + ) + } + + func testForwardSlashRegex24() { + AssertParse( + """ + _ = 1️⃣!/ / + """, + diagnostics: [ + // TODO: Old parser expected error on line 1: regex literal may not start with space; add backslash to escape + // TODO: Old parser expected error on line 1: cannot convert value of type 'Regex' to expected argument type 'Bool' + DiagnosticSpec(message: "expected expression"), + DiagnosticSpec(message: "extraneous code '!/ /' at top level"), + ] + ) + } + + func testForwardSlashRegex25() { + AssertParse( + """ + _ = 1️⃣!!/ / + """, + diagnostics: [ + // TODO: Old parser expected error on line 1: regex literal may not start with space; add backslash to escape + DiagnosticSpec(message: "expected expression"), + DiagnosticSpec(message: "extraneous code '!!/ /' at top level"), + ] + ) + } + + func testForwardSlashRegex26() { + AssertParse( + """ + _ = !!/x/ + """ + ) + } + + func testForwardSlashRegex27() { + AssertParse( + """ + _ = (!!/x/) + """ + ) + } + + func testForwardSlashRegex28() { + AssertParse( + """ + _ = 1️⃣/^) + """, + diagnostics: [ + // TODO: Old parser expected error on line 1: unterminated regex literal + // TODO: Old parser expected error on line 1: closing ')' does not balance any groups openings + DiagnosticSpec(message: "expected expression"), + DiagnosticSpec(message: "extraneous code '/^)' at top level"), + ] + ) + } + + func testForwardSlashRegex29() { + AssertParse( + """ + _ = /x/! + """, + diagnostics: [ + // TODO: Old parser expected error on line 1: cannot force unwrap value of non-optional type 'Regex' + ] + ) + } + + func testForwardSlashRegex30() { + AssertParse( + """ + _ = /x/ + /y/ + """, + diagnostics: [ + // TODO: Old parser expected error on line 1: binary operator '+' cannot be applied to two 'Regex' operands + ] + ) + } + + func testForwardSlashRegex31() { + AssertParse( + """ + _ = /x/+/y/ + """, + diagnostics: [ + // TODO: Old parser expected error on line 1: cannot find operator '+/' in scope + // TODO: Old parser expected error on line 1: '/' is not a postfix unary operator + ] + ) + } + + func testForwardSlashRegex32() { + AssertParse( + """ + _ = /x/?.blah + """, + diagnostics: [ + // TODO: Old parser expected error on line 1: cannot use optional chaining on non-optional value of type 'Regex' + // TODO: Old parser expected error on line 1: value of type 'Regex' has no member 'blah' + ] + ) + } + + func testForwardSlashRegex33() { + AssertParse( + """ + _ = /x/!.blah + """, + diagnostics: [ + // TODO: Old parser expected error on line 1: cannot force unwrap value of non-optional type 'Regex' + // TODO: Old parser expected error on line 1: value of type 'Regex' has no member 'blah' + ] + ) + } + + func testForwardSlashRegex34() { + AssertParse( + """ + do { + _ = /x /? + .blah + } + """, + diagnostics: [ + // TODO: Old parser expected error on line 1: cannot find operator '/?' in scope + // TODO: Old parser expected error on line 1: '/' is not a prefix unary operator + // TODO: Old parser expected error on line 3: cannot infer contextual base in reference to member 'blah' + ] + ) + } + + func testForwardSlashRegex35() { + AssertParse( + """ + _ = /x/? + .blah + """, + diagnostics: [ + // TODO: Old parser expected error on line 1: cannot use optional chaining on non-optional value of type 'Regex' + // TODO: Old parser expected error on line 2: value of type 'Regex' has no member 'blah' + ] + ) + } + + func testForwardSlashRegex36() { + AssertParse( + """ + _ = 0; /x/ + """, + diagnostics: [ + // TODO: Old parser expected warning on line 1: regular expression literal is unused + ] + ) + } + + func testForwardSlashRegex37() { + AssertParse( + """ + do { + _ = 0; /x /1️⃣ + } + """, + diagnostics: [ + DiagnosticSpec(message: "expected expression in 'do' statement"), + // TODO: Old parser expected error on line 3: expected expression after operator + ] + ) + } + + func testForwardSlashRegex38() { + AssertParse( + """ + _ = /x/ ? 0 : 1 + do { + _ = /x / 1️⃣? 0 : 1 + } + """, + diagnostics: [ + // TODO: Old parser expected error on line 1: cannot convert value of type 'Regex' to expected condition type 'Bool' + DiagnosticSpec(message: "expected expression in 'do' statement"), + DiagnosticSpec(message: "unexpected code '? 0 : 1' in 'do' statement"), + // TODO: Old parser expected error on line 3: expected expression after operator + ] + ) + } + + func testForwardSlashRegex39() { + AssertParse( + """ + _ = .random() ? /x/ : .blah + """, + diagnostics: [ + // TODO: Old parser expected error on line 1: type 'Regex' has no member 'blah' + ] + ) + } + + func testForwardSlashRegex40() { + AssertParse( + """ + _ = /x/ ?? /x/ + do { + _ = /x / 1️⃣?? /x / + } + """, + diagnostics: [ + // TODO: Old parser expected warning on line 1: left side of nil coalescing operator '??' has non-optional type 'Regex', so the right side is never used + // TODO: Old parser expected error on line 3: unary operator cannot be separated from its operand + DiagnosticSpec(message: "expected expression in 'do' statement"), + DiagnosticSpec(message: "unexpected code '?? /x /' in 'do' statement"), + // TODO: Old parser expected error on line 4: expected expression after operator + ] + ) + } + + func testForwardSlashRegex41() { + AssertParse( + """ + _ = /x/??/x/ + """, + diagnostics: [ + // TODO: Old parser expected error on line 1: '/' is not a postfix unary operator + ] + ) + } + + func testForwardSlashRegex42() { + AssertParse( + """ + _ = /x/ ... /y/ + """, + diagnostics: [ + // TODO: Old parser expected error on line 1: referencing operator function '...' on 'Comparable' requires that 'Regex' conform to 'Comparable' + ] + ) + } + + func testForwardSlashRegex43() { + AssertParse( + """ + _ = /x/.../y/ + """, + diagnostics: [ + // TODO: Old parser expected error on line 1: missing whitespace between '...' and '/' operators + // TODO: Old parser expected error on line 1: '/' is not a postfix unary operator + ] + ) + } + + func testForwardSlashRegex44() { + AssertParse( + """ + _ = /x/... + """, + diagnostics: [ + // TODO: Old parser expected error on line 1: unary operator '...' cannot be applied to an operand of type 'Regex' + // TODO: Old parser expected note on line 1: overloads for '...' exist with these partially matching parameter lists + ] + ) + } + + func testForwardSlashRegex45() { + AssertParse( + """ + do { + _ = /x1️⃣ /2️⃣... + } + """, + diagnostics: [ + // TODO: Old parser expected error on line 2: '/' is not a prefix unary operator + // TODO: Old parser expected error on line 2: consecutive statements on a line must be separated by ';' + // TODO: Old parser expected error on line 2: operator with postfix spacing cannot start a subexpression + DiagnosticSpec(locationMarker: "1️⃣", message: "consecutive statements on a line must be separated by ';'"), + DiagnosticSpec(locationMarker: "2️⃣", message: "expected expression in prefix operator expression"), + DiagnosticSpec(locationMarker: "2️⃣", message: "unexpected code '...' in 'do' statement"), + // TODO: Old parser expected error on line 3: expected expression + ] + ) + } + + func testForwardSlashRegex46() { + AssertParse( + """ + do { + _ = true / false /1️⃣; + } + """, + diagnostics: [ + // TODO: Old parser expected error on line 2: expected expression after operator + DiagnosticSpec(message: "expected expression in 'do' statement"), + ] + ) + } + + func testForwardSlashRegex47() { + AssertParse( + #""" + _ = "\(/x/)" + """# + ) + } + + func testForwardSlashRegex48() { + AssertParse( + """ + func defaulted(x: Regex = /x/) {} + """ + ) + } + + func testForwardSlashRegex49() { + AssertParse( + """ + func foo(_ x: T, y: T) {} + """, + diagnostics: [ + // TODO: Old parser expected note on line 1: 'foo(_:y:)' declared here + ] + ) + } + + func testForwardSlashRegex50() { + AssertParse( + """ + foo(/abc/, y: /abc/) + """ + ) + } + + func testForwardSlashRegex51() { + AssertParse( + """ + // TODO: The parser ought to have better recovery in cases where a binary + // operator chain is missing an operand, currently we throw everything away. + foo(/abc/, y: /abc /1️⃣) + """, + diagnostics: [ + // TODO: Old parser expected error on line 3: missing argument for parameter 'y' in call + DiagnosticSpec(message: "expected expression in function call"), + // TODO: Old parser expected error on line 3: expected expression after operator + ] + ) + } + + func testForwardSlashRegex52() { + AssertParse( + """ + func bar(_ x: inout T) {} + """ + ) + } + + func testForwardSlashRegex53() { + AssertParse( + """ + bar(&/x/) + """, + diagnostics: [ + // TODO: Old parser expected error on line 1: cannot pass immutable value as inout argument: literals are not mutable + ] + ) + } + + func testForwardSlashRegex54() { + AssertParse( + """ + struct S { + subscript(x: Regex) -> Void { () } + subscript(fn: (Int, Int) -> Int) -> Int { 0 } + } + """, + diagnostics: [ + // TODO: Old parser expected note on line 2: 'subscript(_:)' declared here + ] + ) + } + + func testForwardSlashRegex55() { + AssertParse( + """ + func testSubscript(_ x: S) { + x[/x/] + x[/x /1️⃣] + _ = x[/] / 2 + } + """, + diagnostics: [ + // TODO: Old parser expected error on line 3: missing argument for parameter #1 in call + DiagnosticSpec(message: "expected expression in subscript"), + // TODO: Old parser expected error on line 3: expected expression after operator + ] + ) + } + + func testForwardSlashRegex56() { + AssertParse( + """ + func testReturn() -> Regex { + if .random() { + return /x/ + } + return /x /1️⃣ + } + """, + diagnostics: [ + DiagnosticSpec(message: "expected expression in 'return' statement"), + // TODO: Old parser expected error on line 6: expected expression after operator + ] + ) + } + + func testForwardSlashRegex57() { + AssertParse( + """ + func testThrow() throws { + throw /x/ + } + """, + diagnostics: [ + // TODO: Old parser expected error on line 2: thrown expression type 'Regex' does not conform to 'Error' + ] + ) + } + + func testForwardSlashRegex58() { + AssertParse( + """ + do { + _ = [/abc/, /abc /1️⃣] + } + """, + diagnostics: [ + DiagnosticSpec(message: "expected expression in array element"), + // TODO: Old parser expected error on line 2: expected expression after operator + ] + ) + } + + func testForwardSlashRegex59() { + AssertParse( + """ + do { + _ = [/abc /1️⃣: /abc /2️⃣] + } + """, + diagnostics: [ + DiagnosticSpec(locationMarker: "1️⃣", message: "expected expression in dictionary element"), + // TODO: Old parser expected error on line 2: expected expression after operator + DiagnosticSpec(locationMarker: "2️⃣", message: "expected expression in dictionary element"), + ] + ) + } + + func testForwardSlashRegex60() { + AssertParse( + """ + _ = [/abc/:/abc/] + """, + diagnostics: [ + // TODO: Old parser expected error on line 1: generic struct 'Dictionary' requires that 'Regex' conform to 'Hashable' + ] + ) + } + + func testForwardSlashRegex61() { + AssertParse( + """ + _ = [/abc/ : /abc/] + """, + diagnostics: [ + // TODO: Old parser expected error on line 1: generic struct 'Dictionary' requires that 'Regex' conform to 'Hashable' + ] + ) + } + + func testForwardSlashRegex62() { + AssertParse( + """ + _ = [/abc/:/abc/] + """, + diagnostics: [ + // TODO: Old parser expected error on line 1: generic struct 'Dictionary' requires that 'Regex' conform to 'Hashable' + ] + ) + } + + func testForwardSlashRegex63() { + AssertParse( + """ + _ = [/abc/: /abc/] + """, + diagnostics: [ + // TODO: Old parser expected error on line 1: generic struct 'Dictionary' requires that 'Regex' conform to 'Hashable' + ] + ) + } + + func testForwardSlashRegex64() { + AssertParse( + """ + _ = (/abc/, /abc/) + """ + ) + } + + func testForwardSlashRegex65() { + AssertParse( + """ + _ = ((/abc/)) + """ + ) + } + + func testForwardSlashRegex66() { + AssertParse( + """ + do { + _ = ((/abc /1️⃣)) + } + """, + diagnostics: [ + DiagnosticSpec(message: "expected expression in tuple"), + // TODO: Old parser expected error on line 2: expected expression after operator + ] + ) + } + + func testForwardSlashRegex67() { + AssertParse( + """ + _ = { /abc/ } + """ + ) + } + + func testForwardSlashRegex68() { + AssertParse( + """ + _ = { + /abc/ + } + """ + ) + } + + func testForwardSlashRegex69() { + AssertParse( + """ + let _: () -> Int = { + 0 + / 1 / + 2 + } + """ + ) + } + + func testForwardSlashRegex70() { + AssertParse( + """ + let _: () -> Int = { + 0 + /1 / + 2 + } + """, + diagnostics: [ + // TODO: Old parser expected error on line 3: '/' is not a prefix unary operator + ] + ) + } + + func testForwardSlashRegex71() { + AssertParse( + """ + _ = { + 0 + /1/ + 2 + } + """, + diagnostics: [ + // TODO: Old parser expected warning on line 2: integer literal is unused + // TODO: Old parser expected warning on line 3: regular expression literal is unused + // TODO: Old parser expected warning on line 4: integer literal is unused + ] + ) + } + + func testForwardSlashRegex72() { + AssertParse( + """ + // Operator chain, as a regex literal may not start with space. + """ + ) + } + + func testForwardSlashRegex73() { + AssertParse( + """ + _ = 2 + / 1 / .bitWidth + """ + ) + } + + func testForwardSlashRegex74() { + AssertParse( + """ + _ = 2 + /1/ .bitWidth + """, + diagnostics: [ + // TODO: Old parser expected error on line 2: value of type 'Regex' has no member 'bitWidth' + ] + ) + } + + func testForwardSlashRegex75() { + AssertParse( + """ + _ = 2 + / 1 / + .bitWidth + """ + ) + } + + func testForwardSlashRegex76() { + AssertParse( + """ + _ = 2 + /1 / + .bitWidth + """, + diagnostics: [ + // TODO: Old parser expected error on line 2: '/' is not a prefix unary operator + ] + ) + } + + func testForwardSlashRegex77() { + AssertParse( + """ + _ = !!/1/ .bitWidth + """, + diagnostics: [ + // TODO: Old parser expected error on line 1: value of type 'Regex' has no member 'bitWidth' + ] + ) + } + + func testForwardSlashRegex78() { + AssertParse( + """ + _ = !!/1 / .bitWidth + """, + diagnostics: [ + // TODO: Old parser expected error on line 1: cannot find operator '!!/' in scope + ] + ) + } + + func testForwardSlashRegex79() { + AssertParse( + """ + let z = + /y/ + """ + ) + } + + func testForwardSlashRegex80() { + AssertParse( + """ + // While '.' is technically an operator character, it seems more likely that + // the user hasn't written the member name yet. + """ + ) + } + + func testForwardSlashRegex81() { + AssertParse( + """ + _ = 01️⃣. / 1 / 2 + """, + diagnostics: [ + // TODO: Old parser expected error on line 1: expected member name following '.' + DiagnosticSpec(message: "extraneous code '. / 1 / 2' at top level"), + ] + ) + } + + func testForwardSlashRegex82() { + AssertParse( + """ + _ = 0 . 1️⃣/ 1 / 2 + """, + diagnostics: [ + // TODO: Old parser expected error on line 1: expected member name following '.' + DiagnosticSpec(message: "expected name in member access"), + ] + ) + } + + func testForwardSlashRegex83() { + AssertParse( + #""" + switch "" { + case _ where /x/: + break + default: + break + } + """#, + diagnostics: [ + // TODO: Old parser expected error on line 2: cannot convert value of type 'Regex' to expected condition type 'Bool' + ] + ) + } + + func testForwardSlashRegex84() { + AssertParse( + """ + do {} catch /x/ {} + """ + ) + } + + func testForwardSlashRegex85() { + AssertParse( + """ + + """, + diagnostics: [ + // TODO: Old parser expected error on line 0: expression pattern of type 'Regex' cannot match values of type 'any Error' + // TODO: Old parser expected error on line 0: binary operator '~=' cannot be applied to two 'any Error' operands + // TODO: Old parser expected warning on line 0: 'catch' block is unreachable because no errors are thrown in 'do' block + ] + ) + } + + func testForwardSlashRegex86() { + AssertParse( + """ + switch /x/ { + default: + break + } + """ + ) + } + + func testForwardSlashRegex87() { + AssertParse( + """ + if /x/ {} + """, + diagnostics: [ + // TODO: Old parser expected error on line 1: cannot convert value of type 'Regex' to expected condition type 'Bool' + ] + ) + } + + func testForwardSlashRegex88() { + AssertParse( + """ + if /x/.smth {} + """, + diagnostics: [ + // TODO: Old parser expected error on line 1: value of type 'Regex' has no member 'smth' + ] + ) + } + + func testForwardSlashRegex89() { + AssertParse( + """ + func testGuard() { + guard /x/ else { return } + } + """, + diagnostics: [ + // TODO: Old parser expected error on line 2: cannot convert value of type 'Regex' to expected condition type 'Bool' + ] + ) + } + + func testForwardSlashRegex90() { + AssertParse( + """ + for x in [0] where /x/ {} + """, + diagnostics: [ + // TODO: Old parser expected error on line 1: cannot convert value of type 'Regex' to expected condition type 'Bool' + ] + ) + } + + func testForwardSlashRegex91() { + AssertParse( + """ + typealias Magic = T + """ + ) + } + + func testForwardSlashRegex92() { + AssertParse( + """ + _ = /x/ as Magic + """ + ) + } + + func testForwardSlashRegex93() { + AssertParse( + """ + _ = /x/ as! String + """, + diagnostics: [ + // TODO: Old parser expected warning on line 1: cast from 'Regex' to unrelated type 'String' always fails + ] + ) + } + + func testForwardSlashRegex94() { + AssertParse( + """ + _ = type(of: /x/) + """ + ) + } + + func testForwardSlashRegex95() { + AssertParse( + """ + do { + let 1️⃣/x/ + } + """, + diagnostics: [ + // TODO: Old parser expected error on line 2: expected pattern + DiagnosticSpec(message: "expected pattern in variable"), + ] + ) + } + + func testForwardSlashRegex96() { + AssertParse( + """ + do { + _ = try /x/; _ = try /x /1️⃣ + } + """, + diagnostics: [ + // TODO: Old parser expected warning on line 2: no calls to throwing functions occur within 'try' expression + DiagnosticSpec(message: "expected expression in 'do' statement"), + // TODO: Old parser expected error on line 3: expected expression after operator + ] + ) + } + + func testForwardSlashRegex97() { + AssertParse( + """ + do { + _ = try? /x/; _ = try? /x /1️⃣ + } + """, + diagnostics: [ + // TODO: Old parser expected warning on line 2: no calls to throwing functions occur within 'try' expression + DiagnosticSpec(message: "expected expression in 'do' statement"), + // TODO: Old parser expected error on line 3: expected expression after operator + ] + ) + } + + func testForwardSlashRegex98() { + AssertParse( + """ + do { + _ = try! /x/; _ = try! /x /1️⃣ + } + """, + diagnostics: [ + // TODO: Old parser expected warning on line 2: no calls to throwing functions occur within 'try' expression + DiagnosticSpec(message: "expected expression in 'do' statement"), + // TODO: Old parser expected error on line 3: expected expression after operator + ] + ) + } + + func testForwardSlashRegex99() { + AssertParse( + """ + _ = await /x/ + """, + diagnostics: [ + // TODO: Old parser expected warning on line 1: no 'async' operations occur within 'await' expression + ] + ) + } + + func testForwardSlashRegex100() { + AssertParse( + """ + /x/ = 0 + /x/() + """, + diagnostics: [ + // TODO: Old parser expected error on line 1: cannot assign to value: literals are not mutable + // TODO: Old parser expected error on line 2: cannot call value of non-function type 'Regex' + ] + ) + } + + func testForwardSlashRegex101() { + AssertParse( + """ + // We treat the following as comments, as it seems more likely the user has + // written a comment and is still in the middle of writing the characters before + // it. + """ + ) + } + + func testForwardSlashRegex102() { + AssertParse( + """ + _ = /x// comment + """, + diagnostics: [ + // TODO: Old parser expected error on line 1: '/' is not a prefix unary operator + ] + ) + } + + func testForwardSlashRegex103() { + AssertParse( + """ + _ = /x // comment + """, + diagnostics: [ + // TODO: Old parser expected error on line 1: '/' is not a prefix unary operator + ] + ) + } + + func testForwardSlashRegex104() { + AssertParse( + """ + _ = /x/*comment*/ + """, + diagnostics: [ + // TODO: Old parser expected error on line 1: '/' is not a prefix unary operator + ] + ) + } + + func testForwardSlashRegex105() { + AssertParse( + """ + // MARK: Unapplied operators + """ + ) + } + + func testForwardSlashRegex106() { + AssertParse( + """ + // These become regex literals, unless last character is space, or are surrounded in parens. + """ + ) + } + + func testForwardSlashRegex107() { + AssertParse( + """ + func baz(_ x: (Int, Int) -> Int, _ y: (Int, Int) -> Int) {} // expected-note 3{{'baz' declared here}} + """ + ) + } + + func testForwardSlashRegex108() { + AssertParse( + """ + baz(/, /) + baz(/,/) + """, + diagnostics: [ + // TODO: Old parser expected error on line 2: cannot convert value of type 'Regex' to expected argument type '(Int, Int) -> Int' + // TODO: Old parser expected error on line 2: missing argument for parameter #2 in call + ] + ) + } + + func testForwardSlashRegex109() { + AssertParse( + """ + baz((/), /) + """ + ) + } + + func testForwardSlashRegex110() { + AssertParse( + """ + baz(/^, /) + baz(/^,/) + """, + diagnostics: [ + // TODO: Old parser expected error on line 2: cannot convert value of type 'Regex' to expected argument type '(Int, Int) -> Int' + // TODO: Old parser expected error on line 2: missing argument for parameter #2 in call + ] + ) + } + + func testForwardSlashRegex111() { + AssertParse( + """ + baz((/^), /) + """ + ) + } + + func testForwardSlashRegex112() { + AssertParse( + """ + baz(^^/, /) + baz(^^/,/) + baz((^^/), /) + """, + diagnostics: [ + // TODO: Old parser expected error on line 2: missing argument for parameter #2 in call + ] + ) + } + + func testForwardSlashRegex113() { + AssertParse( + """ + func bazbaz(_ x: (Int, Int) -> Int, _ y: Int) {} + """ + ) + } + + func testForwardSlashRegex114() { + AssertParse( + """ + bazbaz(/, 0) + bazbaz(^^/, 0) + """ + ) + } + + func testForwardSlashRegex115() { + AssertParse( + """ + func qux(_ x: (Int, Int) -> Int, _ y: T) -> Int { 0 } + """ + ) + } + + func testForwardSlashRegex116() { + AssertParse( + #""" + _ = qux(/, 1) / 2 + do { + _ = qux(/, "(") / 2 + _ = qux(/, "(")/1️⃣2 + } + """#, + diagnostics: [ + // TODO: Old parser expected error on line 4: cannot convert value of type 'Regex<(Substring, Substring)>' to expected argument type '(Int, Int) -> Int' + DiagnosticSpec(message: "expected ')' to end function call"), + // TODO: Old parser expected error on line 4: expected ',' separator + ] + ) + } + + func testForwardSlashRegex117() { + AssertParse( + #""" + _ = qux((/), "(") / 2 + """# + ) + } + + func testForwardSlashRegex118() { + AssertParse( + """ + _ = qux(/, 1) // this comment tests to make sure we don't try and end the regex on the starting '/' of '//'. + """ + ) + } + + func testForwardSlashRegex119() { + AssertParse( + """ + _ = qux(/, 1) /* same thing with a block comment */ + """ + ) + } + + func testForwardSlashRegex120() { + AssertParse( + """ + @discardableResult1️⃣ + """, + diagnostics: [ + DiagnosticSpec(message: "expected declaration after attribute"), + ] + ) + } + + func testForwardSlashRegex121() { + AssertParse( + """ + func quxqux(_ x: (Int, Int) -> Int) -> Int { 0 } + """ + ) + } + + func testForwardSlashRegex122() { + AssertParse( + """ + quxqux(/^/) + quxqux((/^/)) + quxqux({ $0 /^/ $1 }) + """, + diagnostics: [ + // TODO: Old parser expected error on line 1: cannot convert value of type 'Regex' to expected argument type '(Int, Int) -> Int' + // TODO: Old parser expected error on line 2: cannot convert value of type 'Regex' to expected argument type '(Int, Int) -> Int' + ] + ) + } + + func testForwardSlashRegex123() { + AssertParse( + """ + quxqux(!/^/) + """, + diagnostics: [ + // TODO: Old parser expected error on line 1: cannot convert value of type 'Bool' to expected argument type '(Int, Int) -> Int' + // TODO: Old parser expected error on line 1: cannot convert value of type 'Regex' to expected argument type 'Bool' + ] + ) + } + + func testForwardSlashRegex124() { + AssertParse( + """ + quxqux(/^) + """ + ) + } + + func testForwardSlashRegex125() { + AssertParse( + """ + _ = quxqux(/^) / 1 + """ + ) + } + + func testForwardSlashRegex126() { + AssertParse( + """ + let arr: [Double] = [2, 3, 4] + """ + ) + } + + func testForwardSlashRegex127() { + AssertParse( + """ + _ = arr.reduce(1, /) / 3 + """ + ) + } + + func testForwardSlashRegex128() { + AssertParse( + """ + _ = arr.reduce(1, /) + arr.reduce(1, /) + """ + ) + } + + func testForwardSlashRegex129() { + AssertParse( + """ + // MARK: ')' disambiguation behavior + """ + ) + } + + func testForwardSlashRegex130() { + AssertParse( + """ + _ = (/x) + """, + diagnostics: [ + // TODO: Old parser expected error on line 1: '/' is not a prefix unary operator + ] + ) + } + + func testForwardSlashRegex131() { + AssertParse( + """ + _ = (/x)/ + """, + diagnostics: [ + // TODO: Old parser expected error on line 1: '/' is not a prefix unary operator + // TODO: Old parser expected error on line 1: '/' is not a postfix unary operator + ] + ) + } + + func testForwardSlashRegex132() { + AssertParse( + """ + _ = (/[(0)])/ + """, + diagnostics: [ + // TODO: Old parser expected error on line 1: '/' is not a prefix unary operator + // TODO: Old parser expected error on line 1: '/' is not a postfix unary operator + ] + ) + } + + func testForwardSlashRegex133() { + AssertParse( + """ + _ = /[(0)]/ + """ + ) + } + + func testForwardSlashRegex134() { + AssertParse( + """ + _ = /(x)/ + """ + ) + } + + func testForwardSlashRegex135() { + AssertParse( + """ + _ = /[1️⃣)]/ + """, + diagnostics: [ + DiagnosticSpec(message: "expected ']' to end array"), + DiagnosticSpec(message: "extraneous code ')]/' at top level"), + ] + ) + } + + func testForwardSlashRegex136() { + AssertParse( + #""" + _ = /[a\1️⃣]2️⃣)]/ + """#, + diagnostics: [ + DiagnosticSpec(locationMarker: "1️⃣", message: "expected root in key path"), + DiagnosticSpec(locationMarker: "2️⃣", message: "extraneous code ')]/' at top level"), + ] + ) + } + + func testForwardSlashRegex137() { + AssertParse( + """ + _ = /([1️⃣)2️⃣])/ + """, + diagnostics: [ + DiagnosticSpec(locationMarker: "1️⃣", message: "expected ']' to end array"), + DiagnosticSpec(locationMarker: "2️⃣", message: "extraneous code '])/' at top level"), + ] + ) + } + + func testForwardSlashRegex138() { + AssertParse( + """ + _ = 1️⃣/]]][)]/ + """, + diagnostics: [ + DiagnosticSpec(message: "expected expression"), + DiagnosticSpec(message: "extraneous code '/]]][)]/' at top level"), + ] + ) + } + + func testForwardSlashRegex139() { + AssertParse( + """ + _ = 1️⃣/ + """, + diagnostics: [ + // TODO: Old parser expected error on line 1: unterminated regex literal + DiagnosticSpec(message: "expected expression"), + DiagnosticSpec(message: "extraneous code '/' at top level"), + ] + ) + } + + func testForwardSlashRegex140() { + AssertParse( + """ + _ = 1️⃣/) + """, + diagnostics: [ + // TODO: Old parser expected error on line 1: unterminated regex literal + // TODO: Old parser expected error on line 1: closing ')' does not balance any groups openings + DiagnosticSpec(message: "expected expression"), + DiagnosticSpec(message: "extraneous code '/)' at top level"), + ] + ) + } + + func testForwardSlashRegex141() { + AssertParse( + """ + let fn: (Int, Int) -> Int = (/) + """ + ) + } + + func testForwardSlashRegex142() { + AssertParse( + #""" + _ = /\()1️⃣/ + """#, + diagnostics: [ + // TODO: Old parser expected error on line 1: '/' is not a prefix unary operator + // TODO: Old parser expected error on line 1: '/' is not a postfix unary operator + // TODO: Old parser expected error on line 1: invalid component of Swift key path + DiagnosticSpec(message: "extraneous code '/' at top level"), + ] + ) + } + + func testForwardSlashRegex143() { + AssertParse( + #""" + do { + let _: Regex = (/whatever\)/1️⃣ + } + """#, + diagnostics: [ + // TODO: Old parser expected note on line 2: to match this opening '(' + DiagnosticSpec(message: "expected ')' to end tuple"), + // TODO: Old parser expected error on line 3: expected ')' in expression list + ] + ) + } + + func testForwardSlashRegex144() { + AssertParse( + """ + do { + _ = /(()())1️⃣)/ + } + """, + diagnostics: [ + // TODO: Old parser expected error on line 2: '/' is not a prefix unary operator + // TODO: Old parser expected error on line 2: consecutive statements on a line must be separated by ';' + // TODO: Old parser expected error on line 2: expected expression + // TODO: Old parser expected error on line 2: cannot call value of non-function type '()' + DiagnosticSpec(message: "unexpected code ')/' in 'do' statement"), + ] + ) + } + + func testForwardSlashRegex145() { + AssertParse( + """ + do { + _ = /[x]1️⃣)/ + } + """, + diagnostics: [ + // TODO: Old parser expected error on line 2: '/' is not a prefix unary operator + // TODO: Old parser expected error on line 2: consecutive statements on a line must be separated by ';' + // TODO: Old parser expected error on line 2: expected expression + DiagnosticSpec(message: "unexpected code ')/' in 'do' statement"), + ] + ) + } + + func testForwardSlashRegex146() { + AssertParse( + #""" + do { + _ = /[\1️⃣]2️⃣])/ + } + """#, + diagnostics: [ + // TODO: Old parser expected error on line 2: expected expression path in Swift key path + DiagnosticSpec(locationMarker: "1️⃣", message: "expected root in key path"), + DiagnosticSpec(locationMarker: "2️⃣", message: "unexpected code '])/' in 'do' statement"), + ] + ) + } + + func testForwardSlashRegex147() { + AssertParse( + """ + _ = ^/x/ + """, + diagnostics: [ + // TODO: Old parser expected error on line 1: '^' is not a prefix unary operator + ] + ) + } + + func testForwardSlashRegex148() { + AssertParse( + """ + _ = (^/x)/ + """, + diagnostics: [ + // TODO: Old parser expected error on line 1: '/' is not a postfix unary operator + ] + ) + } + + func testForwardSlashRegex149() { + AssertParse( + """ + _ = (!!/x/) + """ + ) + } + + func testForwardSlashRegex150() { + AssertParse( + #""" + _ = ^/"/" + """#, + diagnostics: [ + // TODO: Old parser expected error on line 1: '^' is not a prefix unary operator + // TODO: Old parser expected error on line 1: unterminated string literal + ] + ) + } + + func testForwardSlashRegex151() { + AssertParse( + #""" + _ = ^/"[/" + """#, + diagnostics: [ + // TODO: Old parser expected error on line 1: '^' is not a prefix unary operator + // TODO: Old parser expected error on line 1: unterminated string literal + // TODO: Old parser expected error on line 1: expected custom character class members + ] + ) + } + + func testForwardSlashRegex152() { + AssertParse( + #""" + _ = (^/)("/") + """# + ) + } + + func testForwardSlashRegex153() { + AssertParse( + """ + // MARK: Starting characters + """ + ) + } + + func testForwardSlashRegex154() { + AssertParse( + """ + // Fine. + """ + ) + } + + func testForwardSlashRegex155() { + AssertParse( + """ + _ = /./ + """ + ) + } + + func testForwardSlashRegex156() { + AssertParse( + """ + // You need to escape if you want a regex literal to start with these characters. + """ + ) + } + + func testForwardSlashRegex157() { + AssertParse( + #""" + _ = /\ 1️⃣/2️⃣ + """#, + diagnostics: [ + DiagnosticSpec(locationMarker: "1️⃣", message: "expected root in key path"), + DiagnosticSpec(locationMarker: "2️⃣", message: "expected expression"), + ] + ) + } + + func testForwardSlashRegex158() { + AssertParse( + """ + _ = 1️⃣/ / + """, + diagnostics: [ + // TODO: Old parser expected error on line 1: regex literal may not start with space; add backslash to escape, Fix-It replacements: 6 - 6 = '\' + DiagnosticSpec(message: "expected expression"), + DiagnosticSpec(message: "extraneous code '/ /' at top level"), + ] + ) + } + + func testForwardSlashRegex159() { + AssertParse( + """ + _ = 1️⃣/ / + """, + diagnostics: [ + // TODO: Old parser expected error on line 1: regex literal may not start with space; add backslash to escape, Fix-It replacements: 6 - 6 = '\' + // TODO: Old parser expected error on line 1: regex literal may not end with space; use extended literal instead, Fix-It replacements: 5 - 5 = '#', 9 - 9 = '#' + DiagnosticSpec(message: "expected expression"), + DiagnosticSpec(message: "extraneous code '/ /' at top level"), + ] + ) + } + + func testForwardSlashRegex160() { + AssertParse( + """ + _ = #/ /# + """ + ) + } + + func testForwardSlashRegex161() { + AssertParse( + #""" + _ = /x1️⃣\ 2️⃣/3️⃣ + """#, + diagnostics: [ + DiagnosticSpec(locationMarker: "1️⃣", message: "consecutive statements on a line must be separated by ';'"), + DiagnosticSpec(locationMarker: "2️⃣", message: "expected root in key path"), + DiagnosticSpec(locationMarker: "3️⃣", message: "expected expression"), + ] + ) + } + + func testForwardSlashRegex162() { + AssertParse( + #""" + _ = /\ 1️⃣\ 2️⃣/3️⃣ + """#, + diagnostics: [ + DiagnosticSpec(locationMarker: "1️⃣", message: "expected root in key path"), + DiagnosticSpec(locationMarker: "2️⃣", message: "expected root in key path"), + DiagnosticSpec(locationMarker: "3️⃣", message: "expected expression"), + ] + ) + } + + func testForwardSlashRegex163() { + AssertParse( + """ + // There are intentionally trailing spaces here + """ + ) + } + + func testForwardSlashRegex164() { + AssertParse( + """ + _ = 1️⃣/ + """, + diagnostics: [ + // TODO: Old parser expected error on line 1: unterminated regex literal + // TODO: Old parser expected error on line 1: regex literal may not start with space; add backslash to escape, Fix-It replacements: 6 - 6 = '\' + DiagnosticSpec(message: "expected expression"), + DiagnosticSpec(message: "extraneous code '/' at top level"), + ] + ) + } + + func testForwardSlashRegex165() { + AssertParse( + """ + // There are intentionally trailing spaces here + """ + ) + } + + func testForwardSlashRegex166() { + AssertParse( + """ + _ = 1️⃣/^ + """, + diagnostics: [ + // TODO: Old parser expected error on line 1: unterminated regex literal + DiagnosticSpec(message: "expected expression"), + DiagnosticSpec(message: "extraneous code '/^' at top level"), + ] + ) + } + + func testForwardSlashRegex167() { + AssertParse( + #""" + _ = /\)/ + """# + ) + } + + func testForwardSlashRegex168() { + AssertParse( + """ + _ = 1️⃣/)/ + """, + diagnostics: [ + // TODO: Old parser expected error on line 1: closing ')' does not balance any groups openings + DiagnosticSpec(message: "expected expression"), + DiagnosticSpec(message: "extraneous code '/)/' at top level"), + ] + ) + } + + func testForwardSlashRegex169() { + AssertParse( + """ + _ = /,/ + """ + ) + } + + func testForwardSlashRegex170() { + AssertParse( + """ + _ = /}/ + """ + ) + } + + func testForwardSlashRegex171() { + AssertParse( + """ + _ = /]/ + """ + ) + } + + func testForwardSlashRegex172() { + AssertParse( + """ + _ = /:/ + """ + ) + } + + func testForwardSlashRegex173() { + AssertParse( + """ + _ = /;/ + """ + ) + } + + func testForwardSlashRegex174() { + AssertParse( + """ + // Don't emit diagnostics here, as we re-lex. + """ + ) + } + + func testForwardSlashRegex175() { + AssertParse( + """ + _ = /0xG/ + """ + ) + } + + func testForwardSlashRegex176() { + AssertParse( + """ + _ = /0oG/ + """ + ) + } + + func testForwardSlashRegex177() { + AssertParse( + #""" + _ = /"/ + """# + ) + } + + func testForwardSlashRegex178() { + AssertParse( + """ + _ = /'/ + """ + ) + } + + func testForwardSlashRegex179() { + AssertParse( + """ + _ = /<#placeholder#>/ + """ + ) + } + + func testForwardSlashRegex180() { + AssertParse( + """ + _ = ^^/1️⃣0xG/ + """, + diagnostics: [ + DiagnosticSpec(message: "expected expression in prefix operator expression"), + DiagnosticSpec(message: "extraneous code '0xG/' at top level"), + ] + ) + } + + func testForwardSlashRegex181() { + AssertParse( + """ + _ = ^^/1️⃣0oG/ + """, + diagnostics: [ + DiagnosticSpec(message: "expected expression in prefix operator expression"), + DiagnosticSpec(message: "extraneous code '0oG/' at top level"), + ] + ) + } + + func testForwardSlashRegex182() { + AssertParse( + #""" + _ = ^^/"/1️⃣ + """#, + diagnostics: [ + DiagnosticSpec(message: #"expected '"' to end string literal"#), + ] + ) + } + + func testForwardSlashRegex183() { + AssertParse( + """ + _ = ^^/'/1️⃣ + """, + diagnostics: [ + DiagnosticSpec(message: "expected ''' in string literal"), + ] + ) + } + + func testForwardSlashRegex184() { + AssertParse( + """ + _ = ^^/<#placeholder#>/ + """ + ) + } + + func testForwardSlashRegex185() { + AssertParse( + """ + _ = (^^/1️⃣0xG/) + """, + diagnostics: [ + DiagnosticSpec(message: "expected expression in prefix operator expression"), + DiagnosticSpec(message: "unexpected code '0xG/' in tuple"), + ] + ) + } + + func testForwardSlashRegex186() { + AssertParse( + """ + _ = (^^/1️⃣0oG/) + """, + diagnostics: [ + DiagnosticSpec(message: "expected expression in prefix operator expression"), + DiagnosticSpec(message: "unexpected code '0oG/' in tuple"), + ] + ) + } + + func testForwardSlashRegex187() { + AssertParse( + #""" + _ = (^^/"/)1️⃣ + """#, + diagnostics: [ + DiagnosticSpec(message: #"expected '"' to end string literal"#), + DiagnosticSpec(message: "expected ')' to end tuple"), + ] + ) + } + + func testForwardSlashRegex188() { + AssertParse( + """ + _ = (^^/'/)1️⃣ + """, + diagnostics: [ + DiagnosticSpec(message: "expected ''' in string literal"), + DiagnosticSpec(message: "expected ')' to end tuple"), + ] + ) + } + + func testForwardSlashRegex189() { + AssertParse( + """ + _ = (^^/<#placeholder#>/) + """ + ) + } + +} diff --git a/Tests/SwiftParserTest/translated/PrefixSlashTests.swift b/Tests/SwiftParserTest/translated/PrefixSlashTests.swift new file mode 100644 index 00000000000..c9b23853fcd --- /dev/null +++ b/Tests/SwiftParserTest/translated/PrefixSlashTests.swift @@ -0,0 +1,77 @@ +// This test file has been translated from swift/test/StringProcessing/Parse/prefix-slash.swift + +import XCTest + +final class PrefixSlashTests: XCTestCase { + func testPrefixSlash1() { + AssertParse( + """ + // Test the behavior of prefix '/' with regex literals enabled. + """ + ) + } + + func testPrefixSlash2() { + AssertParse( + """ + prefix operator / + prefix func / (_ x: T) -> T { x } + """ + ) + } + + func testPrefixSlash3() { + AssertParse( + """ + enum E { + case e + func foo(_ x: T) {} + } + """ + ) + } + + func testPrefixSlash4() { + AssertParse( + """ + _ = /E.e + (/E.e).foo(/0) + """ + ) + } + + func testPrefixSlash5() { + AssertParse( + """ + func foo(_ x: T, _ y: U) {} + """ + ) + } + + func testPrefixSlash6() { + AssertParse( + """ + foo(/E.e, /E.e) + foo((/E.e), /E.e) + foo((/)(E.e), /E.e) + """ + ) + } + + func testPrefixSlash7() { + AssertParse( + """ + func bar(_ x: T) -> Int { 0 } + """ + ) + } + + func testPrefixSlash8() { + AssertParse( + """ + _ = bar(/E.e) / 2 + """ + ) + } + +} diff --git a/Tests/SwiftParserTest/translated/RegexParseEndOfBufferTests.swift b/Tests/SwiftParserTest/translated/RegexParseEndOfBufferTests.swift new file mode 100644 index 00000000000..49d3bd1f993 --- /dev/null +++ b/Tests/SwiftParserTest/translated/RegexParseEndOfBufferTests.swift @@ -0,0 +1,21 @@ +// This test file has been translated from swift/test/StringProcessing/Parse/regex_parse_end_of_buffer.swift + +import XCTest + +final class RegexParseEndOfBufferTests: XCTestCase { + func testRegexParseEndOfBuffer1() { + AssertParse( + """ + // Note there is purposefully no trailing newline here. + var unterminated = 1️⃣#/(xy2️⃣ + """, + diagnostics: [ + DiagnosticSpec(locationMarker: "1️⃣", message: "use of unknown directive '#'"), + // TODO: Old parser expected error on line 2: unterminated regex literal + DiagnosticSpec(locationMarker: "2️⃣", message: "expected ')' to end tuple"), + // TODO: Old parser expected error on line 2: cannot parse regular expression: expected ')' + ] + ) + } + +} diff --git a/Tests/SwiftParserTest/translated/RegexParseErrorTests.swift b/Tests/SwiftParserTest/translated/RegexParseErrorTests.swift new file mode 100644 index 00000000000..98427f759ed --- /dev/null +++ b/Tests/SwiftParserTest/translated/RegexParseErrorTests.swift @@ -0,0 +1,251 @@ +// This test file has been translated from swift/test/StringProcessing/Parse/regex_parse_error.swift + +import XCTest + +final class RegexParseErrorTests: XCTestCase { + func testRegexParseError1() { + AssertParse( + """ + _ = /(/ + """, + diagnostics: [ + // TODO: Old parser expected error on line 1: expected ')' + ] + ) + } + + func testRegexParseError2() { + AssertParse( + """ + _ = #/(/# + """, + diagnostics: [ + // TODO: Old parser expected error on line 1: expected ')' + ] + ) + } + + func testRegexParseError3() { + AssertParse( + """ + // FIXME: Should be 'group openings' + """ + ) + } + + func testRegexParseError4() { + AssertParse( + """ + _ = 1️⃣/)/ + """, + diagnostics: [ + DiagnosticSpec(message: "expected expression"), + DiagnosticSpec(message: "extraneous code '/)/' at top level"), + // TODO: Old parser expected error on line 1: closing ')' does not balance any groups openings + ] + ) + } + + func testRegexParseError5() { + AssertParse( + """ + _ = 1️⃣#/2️⃣)/# + """, + diagnostics: [ + DiagnosticSpec(locationMarker: "1️⃣", message: "use of unknown directive '#'"), + DiagnosticSpec(locationMarker: "2️⃣", message: "extraneous code ')/#' at top level"), + // TODO: Old parser expected error on line 1: closing ')' does not balance any groups openings + ] + ) + } + + func testRegexParseError6() { + AssertParse( + #""" + _ = 1️⃣#/\2️⃣\3️⃣/''/ + """#, + diagnostics: [ + DiagnosticSpec(locationMarker: "1️⃣", message: "use of unknown directive '#'"), + // TODO: Old parser expected error on line 1: unterminated regex literal + DiagnosticSpec(locationMarker: "2️⃣", message: "expected root in key path"), + DiagnosticSpec(locationMarker: "3️⃣", message: "expected root in key path"), + ] + ) + } + + func testRegexParseError7() { + AssertParse( + #""" + _ = 1️⃣#/\2️⃣| + """#, + diagnostics: [ + DiagnosticSpec(locationMarker: "1️⃣", message: "use of unknown directive '#'"), + // TODO: Old parser expected error on line 1: unterminated regex literal + DiagnosticSpec(locationMarker: "2️⃣", message: "expected root in key path"), + DiagnosticSpec(locationMarker: "2️⃣", message: "extraneous code '|' at top level"), + ] + ) + } + + func testRegexParseError8() { + AssertParse( + """ + _ = 1️⃣#// + """, + diagnostics: [ + DiagnosticSpec(message: "use of unknown directive '#'"), + // TODO: Old parser expected error on line 1: unterminated regex literal + ] + ) + } + + func testRegexParseError9() { + AssertParse( + """ + _ = 1️⃣#/xy + """, + diagnostics: [ + DiagnosticSpec(message: "use of unknown directive '#'"), + // TODO: Old parser expected error on line 1: unterminated regex literal + ] + ) + } + + func testRegexParseError10() { + AssertParse( + """ + _ = #/(?/# + """, + diagnostics: [ + // TODO: Old parser expected error on line 1: expected group specifier + ] + ) + } + + func testRegexParseError11() { + AssertParse( + """ + _ = #/(?'/# + """, + diagnostics: [ + // TODO: Old parser expected error on line 1: expected group name + ] + ) + } + + func testRegexParseError12() { + AssertParse( + """ + _ = #/(?'abc/# + """, + diagnostics: [ + // TODO: Old parser expected error on line 1: expected ''' + ] + ) + } + + func testRegexParseError13() { + AssertParse( + """ + _ = #/(?'abc /# + """, + diagnostics: [ + // TODO: Old parser expected error on line 1: expected ''' + ] + ) + } + + func testRegexParseError14() { + AssertParse( + """ + do { + _ = 1️⃣#/(2️⃣?3️⃣'a + } + """, + diagnostics: [ + DiagnosticSpec(locationMarker: "1️⃣", message: "use of unknown directive '#'"), + // TODO: Old parser expected error on line 2: unterminated regex literal + DiagnosticSpec(locationMarker: "2️⃣", message: "expected value and ')' to end tuple"), + DiagnosticSpec(locationMarker: "3️⃣", message: "expected expression and ':' in 'do' statement"), + DiagnosticSpec(locationMarker: "3️⃣", message: "expected expression in 'do' statement"), + DiagnosticSpec(locationMarker: "3️⃣", message: "unexpected code ''a' in 'do' statement"), + // TODO: Old parser expected error on line 2: cannot parse regular expression: expected ''' + ] + ) + } + + func testRegexParseError15() { + AssertParse( + #""" + _ = #/\(?'abc/# + """# + ) + } + + func testRegexParseError16() { + AssertParse( + #""" + do { + _ = /\1️⃣ + /2️⃣ + } + """#, + diagnostics: [ + DiagnosticSpec(locationMarker: "1️⃣", message: "expected root in key path"), + // TODO: Old parser expected error on line 3: expected expression path in Swift key path + DiagnosticSpec(locationMarker: "2️⃣", message: "expected expression in 'do' statement"), + ] + ) + } + + func testRegexParseError17() { + AssertParse( + #""" + do { + _ = #/\ + /# + } + """#, + diagnostics: [ + // TODO: Old parser expected error on line 2: unterminated regex literal + // TODO: Old parser expected error on line 2: expected escape sequence + // TODO: Old parser expected error on line 3: expected expression + ] + ) + } + + func testRegexParseError18() { + AssertParse( + """ + func foo( + _ x: T, + _ y: T) {} + """ + ) + } + + func testRegexParseError19() { + AssertParse( + """ + foo(#/(?/#, #/abc/#) + foo(#/(?C/#, #/abc/#) + """, + diagnostics: [ + // TODO: Old parser expected error on line 1: expected group specifier + // TODO: Old parser expected error on line 2: expected ')' + ] + ) + } + + func testRegexParseError20() { + AssertParse( + """ + foo(#/(?'/#, #/abc/#) + """, + diagnostics: [ + // TODO: Old parser expected error on line 1: expected group name + ] + ) + } + +} diff --git a/Tests/SwiftParserTest/translated/RegexTests.swift b/Tests/SwiftParserTest/translated/RegexTests.swift new file mode 100644 index 00000000000..102c823a3b7 --- /dev/null +++ b/Tests/SwiftParserTest/translated/RegexTests.swift @@ -0,0 +1,72 @@ +// This test file has been translated from swift/test/StringProcessing/Parse/regex.swift + +import XCTest + +final class RegexTests: XCTestCase { + func testRegex1() { + AssertParse( + """ + _ = /abc/ + _ = #/abc/# + _ = ##/abc/## + """ + ) + } + + func testRegex2() { + AssertParse( + """ + func foo(_ x: T...) {} + """ + ) + } + + func testRegex3() { + AssertParse( + """ + foo(/abc/, #/abc/#, ##/abc/##) + """ + ) + } + + func testRegex4() { + AssertParse( + """ + let arr = [/abc/, #/abc/#, ##/abc/##] + """ + ) + } + + func testRegex5() { + AssertParse( + #""" + _ = /\w+/.self + _ = #/\w+/#.self + _ = ##/\w+/##.self + """# + ) + } + + func testRegex6() { + AssertParse( + ##""" + _ = /#\/\#\\/ + _ = #/#/\/\#\\/# + _ = ##/#|\|\#\\/## + """## + ) + } + + func testRegex7() { + AssertParse( + """ + _ = (#/[*/#, #/+]/#, #/.]/#) + """, + diagnostics: [ + // TODO: Old parser expected error on line 1: cannot parse regular expression: expected ']' + // TODO: Old parser expected error on line 1: cannot parse regular expression: quantifier '+' must appear after expression + ] + ) + } + +} From 4f4750702f0832776a9cff656ef4ca383e4ccf09 Mon Sep 17 00:00:00 2001 From: Hamish Knight Date: Wed, 29 Mar 2023 00:25:21 +0100 Subject: [PATCH 02/12] Cleanup translated tests - Run swift-format over them - Update expectations now that we diagnose unterminated regex literals - Remove semantic errors - Fix test splitting in a few cases - Add TODOs where we should error --- ...orwardSlashRegexSkippingAllowedTests.swift | 35 +- ...orwardSlashRegexSkippingInvalidTests.swift | 119 +-- .../ForwardSlashRegexSkippingTests.swift | 249 ++--- .../translated/ForwardSlashRegexTests.swift | 943 +++++------------- .../translated/PrefixSlashTests.swift | 16 +- .../RegexParseEndOfBufferTests.swift | 9 +- .../translated/RegexParseErrorTests.swift | 134 +-- .../translated/RegexTests.swift | 20 +- 8 files changed, 467 insertions(+), 1058 deletions(-) diff --git a/Tests/SwiftParserTest/translated/ForwardSlashRegexSkippingAllowedTests.swift b/Tests/SwiftParserTest/translated/ForwardSlashRegexSkippingAllowedTests.swift index a743505b0d3..2d1cad195f6 100644 --- a/Tests/SwiftParserTest/translated/ForwardSlashRegexSkippingAllowedTests.swift +++ b/Tests/SwiftParserTest/translated/ForwardSlashRegexSkippingAllowedTests.swift @@ -4,7 +4,7 @@ import XCTest final class ForwardSlashRegexSkippingAllowedTests: XCTestCase { func testForwardSlashRegexSkippingAllowed1() { - AssertParse( + assertParse( """ // Make sure we can skip in all of the below cases. """ @@ -12,7 +12,7 @@ final class ForwardSlashRegexSkippingAllowedTests: XCTestCase { } func testForwardSlashRegexSkippingAllowed2() { - AssertParse( + assertParse( #""" // The printing implementation differs in asserts and no-asserts builds, it will // either print `"Parse.NumFunctionsParsed" 0` or not print it at all. Make sure @@ -23,54 +23,53 @@ final class ForwardSlashRegexSkippingAllowedTests: XCTestCase { } func testForwardSlashRegexSkippingAllowed3() { - AssertParse( + assertParse( """ // Ensures there is a parse error var 1️⃣: Int """, diagnostics: [ - // TODO: Old parser expected error on line 2: expected pattern - DiagnosticSpec(message: "expected pattern in variable"), + DiagnosticSpec(message: "expected pattern in variable") ] ) } func testForwardSlashRegexSkippingAllowed4() { - AssertParse( + assertParse( """ // Balanced `{}`, so okay. func a() { 1️⃣/ {}/ } """, diagnostics: [ - DiagnosticSpec(message: "unexpected code '/ {}/' in function"), + DiagnosticSpec(message: "unexpected code '/ {}/' in function") ] ) } func testForwardSlashRegexSkippingAllowed5() { - AssertParse( + assertParse( #""" func b() { 1️⃣/ \{}/ } """#, diagnostics: [ - DiagnosticSpec(message: #"unexpected code '/ \{}/' in function"#), + DiagnosticSpec(message: #"unexpected code '/ \{}/' in function"#) ] ) } func testForwardSlashRegexSkippingAllowed6() { - AssertParse( + assertParse( #""" func c() { 1️⃣/ {"{"}/ } """#, diagnostics: [ - DiagnosticSpec(message: #"unexpected code '/ {"{"}/' in function"#), + DiagnosticSpec(message: #"unexpected code '/ {"{"}/' in function"#) ] ) } func testForwardSlashRegexSkippingAllowed7() { - AssertParse( + assertParse( """ // Some cases of infix '/' that we should continue to skip. func d() { @@ -82,7 +81,7 @@ final class ForwardSlashRegexSkippingAllowedTests: XCTestCase { } func testForwardSlashRegexSkippingAllowed8() { - AssertParse( + assertParse( #""" func e() { let arr = [1, 2, 3] @@ -95,7 +94,7 @@ final class ForwardSlashRegexSkippingAllowedTests: XCTestCase { } func testForwardSlashRegexSkippingAllowed9() { - AssertParse( + assertParse( """ // Some cases of prefix '/' that we should continue to skip. prefix operator / @@ -105,7 +104,7 @@ final class ForwardSlashRegexSkippingAllowedTests: XCTestCase { } func testForwardSlashRegexSkippingAllowed10() { - AssertParse( + assertParse( """ enum E { case e @@ -116,7 +115,7 @@ final class ForwardSlashRegexSkippingAllowedTests: XCTestCase { } func testForwardSlashRegexSkippingAllowed11() { - AssertParse( + assertParse( """ func f() { _ = /E.e @@ -133,7 +132,7 @@ final class ForwardSlashRegexSkippingAllowedTests: XCTestCase { } func testForwardSlashRegexSkippingAllowed12() { - AssertParse( + assertParse( """ postfix operator / prefix func / (_ x: T) -> T { x } @@ -142,7 +141,7 @@ final class ForwardSlashRegexSkippingAllowedTests: XCTestCase { } func testForwardSlashRegexSkippingAllowed13() { - AssertParse( + assertParse( """ // Some cases of postfix '/' that we should continue to skip. func g() { diff --git a/Tests/SwiftParserTest/translated/ForwardSlashRegexSkippingInvalidTests.swift b/Tests/SwiftParserTest/translated/ForwardSlashRegexSkippingInvalidTests.swift index 9136dfdd3df..31f9577e7aa 100644 --- a/Tests/SwiftParserTest/translated/ForwardSlashRegexSkippingInvalidTests.swift +++ b/Tests/SwiftParserTest/translated/ForwardSlashRegexSkippingInvalidTests.swift @@ -4,7 +4,7 @@ import XCTest final class ForwardSlashRegexSkippingInvalidTests: XCTestCase { func testForwardSlashRegexSkippingInvalid1() { - AssertParse( + assertParse( """ // We don't consider this a regex literal when skipping as it has an initial // space. @@ -19,7 +19,7 @@ final class ForwardSlashRegexSkippingInvalidTests: XCTestCase { } func testForwardSlashRegexSkippingInvalid2() { - AssertParse( + assertParse( """ // Same because of unbalanced ')' func b() { _ = /x1️⃣)*/ } @@ -32,7 +32,7 @@ final class ForwardSlashRegexSkippingInvalidTests: XCTestCase { } func testForwardSlashRegexSkippingInvalid3() { - AssertParse( + assertParse( """ // These also fail the heuristic, but have unbalanced `{` `}`, so we don't skip. func c() { _ = 1️⃣/ x}2️⃣*/ } @@ -52,83 +52,81 @@ final class ForwardSlashRegexSkippingInvalidTests: XCTestCase { } func testForwardSlashRegexSkippingInvalid4() { - AssertParse( + assertParse( """ // Unterminated, and unbalanced `{}`. func e() { _ = 1️⃣/ } + 2️⃣} """, diagnostics: [ - DiagnosticSpec(message: "expected expression in function"), - DiagnosticSpec(message: "unexpected code '/' in function"), + DiagnosticSpec(locationMarker: "1️⃣", message: "expected expression in function"), + DiagnosticSpec(locationMarker: "1️⃣", message: "unexpected code '/' in function"), + DiagnosticSpec(locationMarker: "2️⃣", message: "extraneous brace at top level"), + // TODO: Old parser expected error on line 0: unterminated regex literal + // TODO: Old parser expected error on line 0: regex literal may not start with space; add backslash to escape ] ) } func testForwardSlashRegexSkippingInvalid5() { - AssertParse( + assertParse( """ - 1️⃣} func f() { - _ = 2️⃣/ { + _ = 1️⃣/ { } """, diagnostics: [ - // TODO: Old parser expected error on line 0: unterminated regex literal - // TODO: Old parser expected error on line 0: regex literal may not start with space; add backslash to escape - DiagnosticSpec(locationMarker: "1️⃣", message: "unexpected brace before function"), // TODO: Old parser expected error on line 3: unterminated regex literal // TODO: Old parser expected error on line 3: regex literal may not start with space; add backslash to escape - DiagnosticSpec(locationMarker: "2️⃣", message: "expected expression in function"), - DiagnosticSpec(locationMarker: "2️⃣", message: "expected '}' to end function"), - DiagnosticSpec(locationMarker: "2️⃣", message: "extraneous code at top level"), + DiagnosticSpec(message: "expected expression in function"), + DiagnosticSpec(message: "expected '}' to end function"), + DiagnosticSpec(message: "extraneous code at top level"), ] ) } func testForwardSlashRegexSkippingInvalid6() { - AssertParse( + assertParse( """ func g() { _ = /x } - """ + 1️⃣} + """, + diagnostics: [ + // TODO: Old parser expected error on line 1: extraneous '}' at top level + DiagnosticSpec(message: "extraneous brace at top level"), + ] ) } func testForwardSlashRegexSkippingInvalid7() { - AssertParse( + assertParse( """ - 1️⃣} func h() { _ = /x { - } 2️⃣// The above cannot a regex literal so we skip; this `}` is to balance things out. - """, - diagnostics: [ - // TODO: Old parser expected error on line 1: extraneous '}' at top level - DiagnosticSpec(locationMarker: "1️⃣", message: "unexpected brace before function"), - DiagnosticSpec(locationMarker: "2️⃣", message: "expected '}' to end function"), - ] + } + } + """ ) } func testForwardSlashRegexSkippingInvalid8() { - AssertParse( + assertParse( #""" - 1️⃣} func i() { _ = /x 2️⃣"[abc] { } """#, diagnostics: [ - DiagnosticSpec(locationMarker: "1️⃣", message: "unexpected brace before function"), - // TODO: Old parser expected error on line 3: unterminated string literal + // TODO: Old parser expected error on line 2: unterminated string literal DiagnosticSpec(locationMarker: "2️⃣", message: #"unexpected code '"[abc] {' in function"#), ] ) } func testForwardSlashRegexSkippingInvalid9() { - AssertParse( + assertParse( """ func j() { _ = 1️⃣/^ [abc] { @@ -144,7 +142,7 @@ final class ForwardSlashRegexSkippingInvalidTests: XCTestCase { } func testForwardSlashRegexSkippingInvalid10() { - AssertParse( + assertParse( #""" func k() { _ = 1️⃣/^ "[abc] { @@ -159,7 +157,7 @@ final class ForwardSlashRegexSkippingInvalidTests: XCTestCase { } func testForwardSlashRegexSkippingInvalid11() { - AssertParse( + assertParse( """ func l() { _ = 1️⃣/^ } abc { @@ -174,30 +172,31 @@ final class ForwardSlashRegexSkippingInvalidTests: XCTestCase { } func testForwardSlashRegexSkippingInvalid12() { - AssertParse( + assertParse( #""" func m() { _ = 1️⃣/ " } + 2️⃣} """#, diagnostics: [ // TODO: Old parser expected error on line 2: unterminated string literal - DiagnosticSpec(message: "expected expression in function"), - DiagnosticSpec(message: #"unexpected code '/ "' in function"#), + DiagnosticSpec(locationMarker: "1️⃣", message: "expected expression in function"), + DiagnosticSpec(locationMarker: "1️⃣", message: #"unexpected code '/ "' in function"#), + // TODO: Old parser expected error on line 4: extraneous '}' at top level + DiagnosticSpec(locationMarker: "2️⃣", message: "extraneous brace at top level"), + ] ) } func testForwardSlashRegexSkippingInvalid13() { - AssertParse( + assertParse( #""" - 1️⃣} // Unbalanced `}`, make sure we don't consider the string literal `{`. func n() { 2️⃣/ "{"}3️⃣/ } """#, diagnostics: [ - // TODO: Old parser expected error on line 1: extraneous '}' at top level - DiagnosticSpec(locationMarker: "1️⃣", message: "unexpected brace before function"), // TODO: Old parser expected error on line 3: regex literal may not start with space; add backslash to escape DiagnosticSpec(locationMarker: "2️⃣", message: #"unexpected code '/ "{"' in function"#), DiagnosticSpec(locationMarker: "3️⃣", message: "extraneous code '/ }' at top level"), @@ -206,7 +205,7 @@ final class ForwardSlashRegexSkippingInvalidTests: XCTestCase { } func testForwardSlashRegexSkippingInvalid14() { - AssertParse( + assertParse( """ func o() { _ = { @@ -221,7 +220,7 @@ final class ForwardSlashRegexSkippingInvalidTests: XCTestCase { // TODO: Old parser expected error on line 4: consecutive statements on a line must be separated by ';' // TODO: Old parser expected error on line 4: unterminated regex literal // TODO: Old parser expected warning on line 4: regular expression literal is unused - DiagnosticSpec(message: "extraneous code at top level"), + DiagnosticSpec(message: "extraneous code at top level") // TODO: Old parser expected warning on line 5: integer literal is unused // TODO: Old parser expected error on line 6: extraneous '}' at top level // TODO: Old parser expected error on line 7: extraneous '}' at top level @@ -230,36 +229,26 @@ final class ForwardSlashRegexSkippingInvalidTests: XCTestCase { } func testForwardSlashRegexSkippingInvalid15() { - AssertParse( + assertParse( """ func p() { _ = 2 /x} 1️⃣/ + .bitWidth + } """, diagnostics: [ - DiagnosticSpec(message: "extraneous code '/' at top level"), - ] - ) - } - - func testForwardSlashRegexSkippingInvalid16() { - AssertParse( - """ - .bitWidth - 1️⃣} - """, - diagnostics: [ - // TODO: Old parser expected error on line 0: consecutive statements on a line must be separated by ';' - // TODO: Old parser expected error on line 0: unterminated regex literal - // TODO: Old parser expected error on line 1: value of type 'Regex' has no member 'bitWidth' - // TODO: Old parser expected error on line 2: extraneous '}' at top level - DiagnosticSpec(message: "extraneous brace at top level"), + // TODO: Old parser expected error on line 3: consecutive statements on a line must be separated by ';' + // TODO: Old parser expected error on line 3: unterminated regex literal + // TODO: Old parser expected error on line 4: value of type 'Regex' has no member 'bitWidth' + // TODO: Old parser expected error on line 5: extraneous '}' at top level + DiagnosticSpec(message: "extraneous code at top level") ] ) } func testForwardSlashRegexSkippingInvalid17() { - AssertParse( + assertParse( """ func err1() { _ = 1️⃣/ 0xG}2️⃣/ } """, @@ -273,7 +262,7 @@ final class ForwardSlashRegexSkippingInvalidTests: XCTestCase { } func testForwardSlashRegexSkippingInvalid18() { - AssertParse( + assertParse( """ func err2() { _ = 1️⃣/ 0oG}2️⃣/ } """, @@ -287,7 +276,7 @@ final class ForwardSlashRegexSkippingInvalidTests: XCTestCase { } func testForwardSlashRegexSkippingInvalid19() { - AssertParse( + assertParse( #""" func err3() { _ = 1️⃣/ {"/ } """#, @@ -301,7 +290,7 @@ final class ForwardSlashRegexSkippingInvalidTests: XCTestCase { } func testForwardSlashRegexSkippingInvalid20() { - AssertParse( + assertParse( """ func err4() { _ = 1️⃣/ {'/ } """, @@ -315,7 +304,7 @@ final class ForwardSlashRegexSkippingInvalidTests: XCTestCase { } func testForwardSlashRegexSkippingInvalid21() { - AssertParse( + assertParse( """ func err5() { _ = 1️⃣/ {<#placeholder#>/ } """, diff --git a/Tests/SwiftParserTest/translated/ForwardSlashRegexSkippingTests.swift b/Tests/SwiftParserTest/translated/ForwardSlashRegexSkippingTests.swift index a5294a20a55..9933da12055 100644 --- a/Tests/SwiftParserTest/translated/ForwardSlashRegexSkippingTests.swift +++ b/Tests/SwiftParserTest/translated/ForwardSlashRegexSkippingTests.swift @@ -3,27 +3,8 @@ import XCTest final class ForwardSlashRegexSkippingTests: XCTestCase { - func testForwardSlashRegexSkipping1() { - AssertParse( - """ - // Make sure we properly handle `/.../` regex literals in skipped function - // bodies. Currently we detect them and avoid skipping, but in the future we - // ought to be able to skip over them. - """ - ) - } - - func testForwardSlashRegexSkipping2() { - AssertParse( - """ - prefix operator ^^ - prefix func ^^ (_ x: T) -> T { x } - """ - ) - } - func testForwardSlashRegexSkipping3() { - AssertParse( + assertParse( #""" struct A { static let r = /test":"(.*?)"/ @@ -33,7 +14,7 @@ final class ForwardSlashRegexSkippingTests: XCTestCase { } func testForwardSlashRegexSkipping4() { - AssertParse( + assertParse( """ struct B { static let r = /x*/ @@ -43,7 +24,7 @@ final class ForwardSlashRegexSkippingTests: XCTestCase { } func testForwardSlashRegexSkipping5() { - AssertParse( + assertParse( """ struct C { func foo() { @@ -55,36 +36,29 @@ final class ForwardSlashRegexSkippingTests: XCTestCase { } func testForwardSlashRegexSkipping6() { - AssertParse( + assertParse( """ struct D { func foo() { func bar() { let r = /x}}*/ - }1️⃣ - """, - diagnostics: [ - DiagnosticSpec(message: "expected '}' to end function"), - DiagnosticSpec(message: "expected '}' to end struct"), - ] + } + } + } + """ ) } func testForwardSlashRegexSkipping7() { - AssertParse( + assertParse( """ - 1️⃣} - } func a() { _ = /abc}}*/ } - """, - diagnostics: [ - DiagnosticSpec(message: "unexpected braces before function"), - ] + """ ) } func testForwardSlashRegexSkipping8() { - AssertParse( + assertParse( #""" func b() { _ = /\// } """# @@ -92,7 +66,7 @@ final class ForwardSlashRegexSkippingTests: XCTestCase { } func testForwardSlashRegexSkipping9() { - AssertParse( + assertParse( #""" func c() { _ = /\\/ } """# @@ -100,22 +74,24 @@ final class ForwardSlashRegexSkippingTests: XCTestCase { } func testForwardSlashRegexSkipping10() { - AssertParse( + assertParse( """ func d() { _ = ^^/x}1️⃣}*/ } """, diagnostics: [ + // FIXME: This should not emit any diagnostics. DiagnosticSpec(message: "extraneous code '}*/ }' at top level"), ] ) } func testForwardSlashRegexSkipping11() { - AssertParse( + assertParse( """ func e() { _ = (^^/x1️⃣}2️⃣}*/) } """, diagnostics: [ + // FIXME: This should not emit any diagnostics. DiagnosticSpec(locationMarker: "1️⃣", message: "expected ')' to end tuple"), DiagnosticSpec(locationMarker: "2️⃣", message: "extraneous code '}*/) }' at top level"), ] @@ -123,18 +99,19 @@ final class ForwardSlashRegexSkippingTests: XCTestCase { } func testForwardSlashRegexSkipping12() { - AssertParse( + assertParse( """ func f() { _ = ^^/^x}1️⃣}*/ } """, diagnostics: [ + // FIXME: This should not emit any diagnostics. DiagnosticSpec(message: "extraneous code '}*/ }' at top level"), ] ) } func testForwardSlashRegexSkipping13() { - AssertParse( + assertParse( #""" func g() { _ = "\(/x}}*/)" } """# @@ -142,18 +119,19 @@ final class ForwardSlashRegexSkippingTests: XCTestCase { } func testForwardSlashRegexSkipping14() { - AssertParse( + assertParse( #""" func h() { _ = "\(^^/x1️⃣}}*/)" } """#, diagnostics: [ + // FIXME: This should not emit any diagnostics. DiagnosticSpec(message: "unexpected code '}}*/' in string literal"), ] ) } func testForwardSlashRegexSkipping15() { - AssertParse( + assertParse( #""" func i() { func foo(_ x: T, y: T) {} @@ -164,7 +142,7 @@ final class ForwardSlashRegexSkippingTests: XCTestCase { } func testForwardSlashRegexSkipping16() { - AssertParse( + assertParse( """ func j() { _ = { @@ -178,7 +156,7 @@ final class ForwardSlashRegexSkippingTests: XCTestCase { } func testForwardSlashRegexSkipping17() { - AssertParse( + assertParse( """ func k() { _ = 2 @@ -189,31 +167,18 @@ final class ForwardSlashRegexSkippingTests: XCTestCase { } func testForwardSlashRegexSkipping18() { - AssertParse( + assertParse( """ func l() { _ = 2 - /x}*/ .self1️⃣ - """, - diagnostics: [ - DiagnosticSpec(message: "expected '}' to end function"), - ] - ) - } - - func testForwardSlashRegexSkipping19() { - AssertParse( + /x}*/ .self + } """ - 1️⃣} - """, - diagnostics: [ - DiagnosticSpec(message: "extraneous brace at top level"), - ] ) } func testForwardSlashRegexSkipping20() { - AssertParse( + assertParse( """ func m() { _ = 2 @@ -225,32 +190,19 @@ final class ForwardSlashRegexSkippingTests: XCTestCase { } func testForwardSlashRegexSkipping21() { - AssertParse( + assertParse( """ func n() { _ = 2 - /x}/1️⃣ - """, - diagnostics: [ - DiagnosticSpec(message: "expected '}' to end function"), - ] - ) - } - - func testForwardSlashRegexSkipping22() { - AssertParse( + /x}/ + .bitWidth + } """ - .bitWidth - 1️⃣} - """, - diagnostics: [ - DiagnosticSpec(message: "extraneous brace at top level"), - ] ) } func testForwardSlashRegexSkipping23() { - AssertParse( + assertParse( """ func o() { _ = /x// comment @@ -260,7 +212,7 @@ final class ForwardSlashRegexSkippingTests: XCTestCase { } func testForwardSlashRegexSkipping24() { - AssertParse( + assertParse( """ func p() { _ = /x // comment @@ -270,7 +222,7 @@ final class ForwardSlashRegexSkippingTests: XCTestCase { } func testForwardSlashRegexSkipping25() { - AssertParse( + assertParse( """ func q() { _ = /x/*comment*/ @@ -280,7 +232,7 @@ final class ForwardSlashRegexSkippingTests: XCTestCase { } func testForwardSlashRegexSkipping26() { - AssertParse( + assertParse( """ func r() { _ = /[(0)]/ } """ @@ -288,7 +240,7 @@ final class ForwardSlashRegexSkippingTests: XCTestCase { } func testForwardSlashRegexSkipping27() { - AssertParse( + assertParse( """ func s() { _ = /(x)/ } """ @@ -296,56 +248,39 @@ final class ForwardSlashRegexSkippingTests: XCTestCase { } func testForwardSlashRegexSkipping28() { - AssertParse( + assertParse( + """ + func t() { _ = /[)]/ } """ - func t() { _ = /[1️⃣)]/ } - """, - diagnostics: [ - DiagnosticSpec(message: "expected ']' to end array"), - DiagnosticSpec(message: "unexpected code ')]/' in function"), - ] ) } func testForwardSlashRegexSkipping29() { - AssertParse( + assertParse( #""" - func u() { _ = /[a\1️⃣]2️⃣)]/ } - """#, - diagnostics: [ - DiagnosticSpec(locationMarker: "1️⃣", message: "expected root in key path"), - DiagnosticSpec(locationMarker: "2️⃣", message: "unexpected code ')]/' in function"), - ] + func u() { _ = /[a\])]/ } + """# ) } func testForwardSlashRegexSkipping30() { - AssertParse( + assertParse( + """ + func v() { _ = /([)])/ } """ - func v() { _ = /([1️⃣)2️⃣])/ } - """, - diagnostics: [ - DiagnosticSpec(locationMarker: "1️⃣", message: "expected ']' to end array"), - DiagnosticSpec(locationMarker: "2️⃣", message: "unexpected code '])/' in function"), - ] ) } func testForwardSlashRegexSkipping31() { - AssertParse( + assertParse( + """ + func w() { _ = /]]][)]/ } """ - func w() { _ = 1️⃣/]]][)]/ } - """, - diagnostics: [ - DiagnosticSpec(message: "expected expression in function"), - DiagnosticSpec(message: "expected '}' to end function"), - DiagnosticSpec(message: "extraneous code '/]]][)]/ }' at top level"), - ] ) } func testForwardSlashRegexSkipping32() { - AssertParse( + assertParse( """ func x() { _ = /,/ } """ @@ -353,7 +288,7 @@ final class ForwardSlashRegexSkippingTests: XCTestCase { } func testForwardSlashRegexSkipping33() { - AssertParse( + assertParse( """ func y() { _ = /}/ } """ @@ -361,7 +296,7 @@ final class ForwardSlashRegexSkippingTests: XCTestCase { } func testForwardSlashRegexSkipping34() { - AssertParse( + assertParse( """ func z() { _ = /]/ } """ @@ -369,7 +304,7 @@ final class ForwardSlashRegexSkippingTests: XCTestCase { } func testForwardSlashRegexSkipping35() { - AssertParse( + assertParse( """ func a1() { _ = /:/ } """ @@ -377,7 +312,7 @@ final class ForwardSlashRegexSkippingTests: XCTestCase { } func testForwardSlashRegexSkipping36() { - AssertParse( + assertParse( """ func a2() { _ = /;/ } """ @@ -385,11 +320,13 @@ final class ForwardSlashRegexSkippingTests: XCTestCase { } func testForwardSlashRegexSkipping37() { - AssertParse( + assertParse( """ func a3() { _ = 1️⃣/)/ } """, diagnostics: [ + // FIXME: Improve diagnostics (ideally we'd be able to parse as an + // invalid regex) DiagnosticSpec(message: "expected expression in function"), DiagnosticSpec(message: "unexpected code '/)/' in function"), ] @@ -397,7 +334,7 @@ final class ForwardSlashRegexSkippingTests: XCTestCase { } func testForwardSlashRegexSkipping38() { - AssertParse( + assertParse( """ func a4() { _ = 1️⃣/ / } """, @@ -410,39 +347,15 @@ final class ForwardSlashRegexSkippingTests: XCTestCase { } func testForwardSlashRegexSkipping39() { - AssertParse( + assertParse( #""" - func a5() { _ = /\ 1️⃣/ 2️⃣} - """#, - diagnostics: [ - DiagnosticSpec(locationMarker: "1️⃣", message: "expected root in key path"), - DiagnosticSpec(locationMarker: "2️⃣", message: "expected expression in function"), - ] - ) - } - - func testForwardSlashRegexSkipping40() { - AssertParse( - """ - prefix operator / - prefix func / (_ x: T) -> T { x } - """ - ) - } - - func testForwardSlashRegexSkipping41() { - AssertParse( - """ - enum E { - case e - func foo(_ x: T) {} - } - """ + func a5() { _ = /\ / } + """# ) } func testForwardSlashRegexSkipping42() { - AssertParse( + assertParse( #""" func a7() { _ = /\/}/ } """# @@ -450,7 +363,7 @@ final class ForwardSlashRegexSkippingTests: XCTestCase { } func testForwardSlashRegexSkipping43() { - AssertParse( + assertParse( """ // Make sure we don't emit errors for these. func err1() { _ = /0xG/ } @@ -459,7 +372,7 @@ final class ForwardSlashRegexSkippingTests: XCTestCase { } func testForwardSlashRegexSkipping44() { - AssertParse( + assertParse( """ func err2() { _ = /0oG/ } """ @@ -467,7 +380,7 @@ final class ForwardSlashRegexSkippingTests: XCTestCase { } func testForwardSlashRegexSkipping45() { - AssertParse( + assertParse( #""" func err3() { _ = /"/ } """# @@ -475,7 +388,7 @@ final class ForwardSlashRegexSkippingTests: XCTestCase { } func testForwardSlashRegexSkipping46() { - AssertParse( + assertParse( """ func err4() { _ = /'/ } """ @@ -483,7 +396,7 @@ final class ForwardSlashRegexSkippingTests: XCTestCase { } func testForwardSlashRegexSkipping47() { - AssertParse( + assertParse( """ func err5() { _ = /<#placeholder#>/ } """ @@ -491,11 +404,12 @@ final class ForwardSlashRegexSkippingTests: XCTestCase { } func testForwardSlashRegexSkipping48() { - AssertParse( + assertParse( """ func err6() { _ = ^^/1️⃣0xG/ } """, diagnostics: [ + // FIXME: This should not emit any diagnostics. DiagnosticSpec(message: "expected expression in prefix operator expression"), DiagnosticSpec(message: "unexpected code '0xG/' in function"), ] @@ -503,11 +417,12 @@ final class ForwardSlashRegexSkippingTests: XCTestCase { } func testForwardSlashRegexSkipping49() { - AssertParse( + assertParse( """ func err7() { _ = ^^/1️⃣0oG/ } """, diagnostics: [ + // FIXME: This should not emit any diagnostics. DiagnosticSpec(message: "expected expression in prefix operator expression"), DiagnosticSpec(message: "unexpected code '0oG/' in function"), ] @@ -515,11 +430,12 @@ final class ForwardSlashRegexSkippingTests: XCTestCase { } func testForwardSlashRegexSkipping50() { - AssertParse( + assertParse( #""" func err8() { _ = ^^/"/ }1️⃣ """#, diagnostics: [ + // FIXME: This should not emit any diagnostics. DiagnosticSpec(message: #"expected '"' to end string literal"#), DiagnosticSpec(message: "expected '}' to end function"), ] @@ -527,11 +443,12 @@ final class ForwardSlashRegexSkippingTests: XCTestCase { } func testForwardSlashRegexSkipping51() { - AssertParse( + assertParse( """ func err9() { _ = ^^/'/ }1️⃣ """, diagnostics: [ + // FIXME: This should not emit any diagnostics. DiagnosticSpec(message: "expected ''' in string literal"), DiagnosticSpec(message: "expected '}' to end function"), ] @@ -539,7 +456,7 @@ final class ForwardSlashRegexSkippingTests: XCTestCase { } func testForwardSlashRegexSkipping52() { - AssertParse( + assertParse( """ func err10() { _ = ^^/<#placeholder#>/ } """ @@ -547,11 +464,12 @@ final class ForwardSlashRegexSkippingTests: XCTestCase { } func testForwardSlashRegexSkipping53() { - AssertParse( + assertParse( """ func err11() { _ = (^^/1️⃣0xG/) } """, diagnostics: [ + // FIXME: This should not emit any diagnostics. DiagnosticSpec(message: "expected expression in prefix operator expression"), DiagnosticSpec(message: "unexpected code '0xG/' in tuple"), ] @@ -559,11 +477,12 @@ final class ForwardSlashRegexSkippingTests: XCTestCase { } func testForwardSlashRegexSkipping54() { - AssertParse( + assertParse( """ func err12() { _ = (^^/1️⃣0oG/) } """, diagnostics: [ + // FIXME: This should not emit any diagnostics. DiagnosticSpec(message: "expected expression in prefix operator expression"), DiagnosticSpec(message: "unexpected code '0oG/' in tuple"), ] @@ -571,11 +490,12 @@ final class ForwardSlashRegexSkippingTests: XCTestCase { } func testForwardSlashRegexSkipping55() { - AssertParse( + assertParse( #""" func err13() { _ = (^^/"/) }1️⃣ """#, diagnostics: [ + // FIXME: This should not emit any diagnostics. DiagnosticSpec(message: #"expected '"' to end string literal"#), DiagnosticSpec(message: "expected ')' to end tuple"), DiagnosticSpec(message: "expected '}' to end function"), @@ -584,11 +504,12 @@ final class ForwardSlashRegexSkippingTests: XCTestCase { } func testForwardSlashRegexSkipping56() { - AssertParse( + assertParse( """ func err14() { _ = (^^/'/) }1️⃣ """, diagnostics: [ + // FIXME: This should not emit any diagnostics. DiagnosticSpec(message: "expected ''' in string literal"), DiagnosticSpec(message: "expected ')' to end tuple"), DiagnosticSpec(message: "expected '}' to end function"), @@ -597,7 +518,7 @@ final class ForwardSlashRegexSkippingTests: XCTestCase { } func testForwardSlashRegexSkipping57() { - AssertParse( + assertParse( """ func err15() { _ = (^^/<#placeholder#>/) } """ diff --git a/Tests/SwiftParserTest/translated/ForwardSlashRegexTests.swift b/Tests/SwiftParserTest/translated/ForwardSlashRegexTests.swift index 69c9cdcd7d3..9e64fd22b5e 100644 --- a/Tests/SwiftParserTest/translated/ForwardSlashRegexTests.swift +++ b/Tests/SwiftParserTest/translated/ForwardSlashRegexTests.swift @@ -4,7 +4,7 @@ import XCTest final class ForwardSlashRegexTests: XCTestCase { func testForwardSlashRegex1() { - AssertParse( + assertParse( """ prefix operator / prefix operator ^/ @@ -14,57 +14,15 @@ final class ForwardSlashRegexTests: XCTestCase { } func testForwardSlashRegex2() { - AssertParse( + assertParse( """ prefix func ^/ (_ x: T) -> T { x } """ ) } - func testForwardSlashRegex3() { - AssertParse( - """ - prefix operator !! - """ - ) - } - - func testForwardSlashRegex4() { - AssertParse( - """ - prefix func !! (_ x: T) -> T { x } - """ - ) - } - - func testForwardSlashRegex5() { - AssertParse( - """ - prefix operator ^^ - """ - ) - } - - func testForwardSlashRegex6() { - AssertParse( - """ - prefix func ^^ (_ x: T) -> T { x } - """ - ) - } - - func testForwardSlashRegex7() { - AssertParse( - """ - precedencegroup P { - associativity: left - } - """ - ) - } - func testForwardSlashRegex8() { - AssertParse( + assertParse( """ // The divisions in the body of the below operators make sure we don't try and // consider them to be ending delimiters of a regex. @@ -75,7 +33,7 @@ final class ForwardSlashRegexTests: XCTestCase { } func testForwardSlashRegex9() { - AssertParse( + assertParse( """ infix operator /^ : P func /^ (lhs: Int, rhs: Int) -> Int { 1 / 2 } @@ -84,7 +42,7 @@ final class ForwardSlashRegexTests: XCTestCase { } func testForwardSlashRegex10() { - AssertParse( + assertParse( """ infix operator ^^/ : P func ^^/ (lhs: Int, rhs: Int) -> Int { 1 / 2 } @@ -93,7 +51,7 @@ final class ForwardSlashRegexTests: XCTestCase { } func testForwardSlashRegex11() { - AssertParse( + assertParse( """ let i = 01️⃣ /^/2️⃣ 1/^/3 """, @@ -105,7 +63,7 @@ final class ForwardSlashRegexTests: XCTestCase { } func testForwardSlashRegex12() { - AssertParse( + assertParse( """ let x = /abc/ """ @@ -113,7 +71,7 @@ final class ForwardSlashRegexTests: XCTestCase { } func testForwardSlashRegex13() { - AssertParse( + assertParse( """ _ = /abc/ """ @@ -121,7 +79,7 @@ final class ForwardSlashRegexTests: XCTestCase { } func testForwardSlashRegex14() { - AssertParse( + assertParse( """ _ = /x/.self """ @@ -129,7 +87,7 @@ final class ForwardSlashRegexTests: XCTestCase { } func testForwardSlashRegex15() { - AssertParse( + assertParse( #""" _ = /\// """# @@ -137,24 +95,15 @@ final class ForwardSlashRegexTests: XCTestCase { } func testForwardSlashRegex16() { - AssertParse( + assertParse( #""" _ = /\\/ """# ) } - func testForwardSlashRegex17() { - AssertParse( - """ - // This is just here to appease typo correction. - let y = 0 - """ - ) - } - func testForwardSlashRegex18() { - AssertParse( + assertParse( """ // These unfortunately become prefix `=` and infix `=/` respectively. We could // likely improve the diagnostic though. @@ -172,69 +121,47 @@ final class ForwardSlashRegexTests: XCTestCase { } func testForwardSlashRegex19() { - AssertParse( + assertParse( """ do { _=/0/ } - """, - diagnostics: [ - // TODO: Old parser expected error on line 2: '_' can only appear in a pattern or on the left side of an assignment - // TODO: Old parser expected error on line 2: cannot find operator '=/' in scope - // TODO: Old parser expected error on line 2: '/' is not a postfix unary operator - ] - ) - } - - func testForwardSlashRegex20() { - AssertParse( - """ - // No closing '/' so a prefix operator. """ ) } func testForwardSlashRegex21() { - AssertParse( + assertParse( """ _ = /x - """, - diagnostics: [ - // TODO: Old parser expected error on line 1: '/' is not a prefix unary operator - ] + """ ) } func testForwardSlashRegex22() { - AssertParse( + assertParse( """ _ = !/x/ - """, - diagnostics: [ - // TODO: Old parser expected error on line 1: cannot convert value of type 'Regex' to expected argument type 'Bool' - ] + """ ) } func testForwardSlashRegex23() { - AssertParse( + assertParse( """ _ = (!/x/) - """, - diagnostics: [ - // TODO: Old parser expected error on line 1: cannot convert value of type 'Regex' to expected argument type 'Bool' - ] + """ ) } func testForwardSlashRegex24() { - AssertParse( + assertParse( """ _ = 1️⃣!/ / """, diagnostics: [ // TODO: Old parser expected error on line 1: regex literal may not start with space; add backslash to escape - // TODO: Old parser expected error on line 1: cannot convert value of type 'Regex' to expected argument type 'Bool' + DiagnosticSpec(message: "expected expression"), DiagnosticSpec(message: "extraneous code '!/ /' at top level"), ] @@ -242,7 +169,7 @@ final class ForwardSlashRegexTests: XCTestCase { } func testForwardSlashRegex25() { - AssertParse( + assertParse( """ _ = 1️⃣!!/ / """, @@ -255,7 +182,7 @@ final class ForwardSlashRegexTests: XCTestCase { } func testForwardSlashRegex26() { - AssertParse( + assertParse( """ _ = !!/x/ """ @@ -263,7 +190,7 @@ final class ForwardSlashRegexTests: XCTestCase { } func testForwardSlashRegex27() { - AssertParse( + assertParse( """ _ = (!!/x/) """ @@ -271,13 +198,12 @@ final class ForwardSlashRegexTests: XCTestCase { } func testForwardSlashRegex28() { - AssertParse( + assertParse( """ _ = 1️⃣/^) """, diagnostics: [ // TODO: Old parser expected error on line 1: unterminated regex literal - // TODO: Old parser expected error on line 1: closing ')' does not balance any groups openings DiagnosticSpec(message: "expected expression"), DiagnosticSpec(message: "extraneous code '/^)' at top level"), ] @@ -285,105 +211,76 @@ final class ForwardSlashRegexTests: XCTestCase { } func testForwardSlashRegex29() { - AssertParse( + assertParse( """ _ = /x/! - """, - diagnostics: [ - // TODO: Old parser expected error on line 1: cannot force unwrap value of non-optional type 'Regex' - ] + """ ) } func testForwardSlashRegex30() { - AssertParse( + assertParse( """ _ = /x/ + /y/ - """, - diagnostics: [ - // TODO: Old parser expected error on line 1: binary operator '+' cannot be applied to two 'Regex' operands - ] + """ ) } func testForwardSlashRegex31() { - AssertParse( + assertParse( """ _ = /x/+/y/ - """, - diagnostics: [ - // TODO: Old parser expected error on line 1: cannot find operator '+/' in scope - // TODO: Old parser expected error on line 1: '/' is not a postfix unary operator - ] + """ ) } func testForwardSlashRegex32() { - AssertParse( + assertParse( """ _ = /x/?.blah - """, - diagnostics: [ - // TODO: Old parser expected error on line 1: cannot use optional chaining on non-optional value of type 'Regex' - // TODO: Old parser expected error on line 1: value of type 'Regex' has no member 'blah' - ] + """ ) } func testForwardSlashRegex33() { - AssertParse( + assertParse( """ _ = /x/!.blah - """, - diagnostics: [ - // TODO: Old parser expected error on line 1: cannot force unwrap value of non-optional type 'Regex' - // TODO: Old parser expected error on line 1: value of type 'Regex' has no member 'blah' - ] + """ ) } func testForwardSlashRegex34() { - AssertParse( + // Regex literals cannot end with space, so this is infix '/?' + assertParse( """ do { _ = /x /? .blah } - """, - diagnostics: [ - // TODO: Old parser expected error on line 1: cannot find operator '/?' in scope - // TODO: Old parser expected error on line 1: '/' is not a prefix unary operator - // TODO: Old parser expected error on line 3: cannot infer contextual base in reference to member 'blah' - ] + """ ) } func testForwardSlashRegex35() { - AssertParse( + assertParse( """ _ = /x/? .blah - """, - diagnostics: [ - // TODO: Old parser expected error on line 1: cannot use optional chaining on non-optional value of type 'Regex' - // TODO: Old parser expected error on line 2: value of type 'Regex' has no member 'blah' - ] + """ ) } func testForwardSlashRegex36() { - AssertParse( + assertParse( """ _ = 0; /x/ - """, - diagnostics: [ - // TODO: Old parser expected warning on line 1: regular expression literal is unused - ] + """ ) } func testForwardSlashRegex37() { - AssertParse( + assertParse( """ do { _ = 0; /x /1️⃣ @@ -397,7 +294,7 @@ final class ForwardSlashRegexTests: XCTestCase { } func testForwardSlashRegex38() { - AssertParse( + assertParse( """ _ = /x/ ? 0 : 1 do { @@ -405,7 +302,6 @@ final class ForwardSlashRegexTests: XCTestCase { } """, diagnostics: [ - // TODO: Old parser expected error on line 1: cannot convert value of type 'Regex' to expected condition type 'Bool' DiagnosticSpec(message: "expected expression in 'do' statement"), DiagnosticSpec(message: "unexpected code '? 0 : 1' in 'do' statement"), // TODO: Old parser expected error on line 3: expected expression after operator @@ -414,18 +310,15 @@ final class ForwardSlashRegexTests: XCTestCase { } func testForwardSlashRegex39() { - AssertParse( + assertParse( """ _ = .random() ? /x/ : .blah - """, - diagnostics: [ - // TODO: Old parser expected error on line 1: type 'Regex' has no member 'blah' - ] + """ ) } func testForwardSlashRegex40() { - AssertParse( + assertParse( """ _ = /x/ ?? /x/ do { @@ -433,7 +326,6 @@ final class ForwardSlashRegexTests: XCTestCase { } """, diagnostics: [ - // TODO: Old parser expected warning on line 1: left side of nil coalescing operator '??' has non-optional type 'Regex', so the right side is never used // TODO: Old parser expected error on line 3: unary operator cannot be separated from its operand DiagnosticSpec(message: "expected expression in 'do' statement"), DiagnosticSpec(message: "unexpected code '?? /x /' in 'do' statement"), @@ -443,72 +335,58 @@ final class ForwardSlashRegexTests: XCTestCase { } func testForwardSlashRegex41() { - AssertParse( + // This parses as /x/ ??/ x/ + assertParse( """ _ = /x/??/x/ - """, - diagnostics: [ - // TODO: Old parser expected error on line 1: '/' is not a postfix unary operator - ] + """ ) } func testForwardSlashRegex42() { - AssertParse( + assertParse( """ _ = /x/ ... /y/ - """, - diagnostics: [ - // TODO: Old parser expected error on line 1: referencing operator function '...' on 'Comparable' requires that 'Regex' conform to 'Comparable' - ] + """ ) } func testForwardSlashRegex43() { - AssertParse( + // This parses as /x/ .../ x/ + assertParse( """ _ = /x/.../y/ - """, - diagnostics: [ - // TODO: Old parser expected error on line 1: missing whitespace between '...' and '/' operators - // TODO: Old parser expected error on line 1: '/' is not a postfix unary operator - ] + """ ) } func testForwardSlashRegex44() { - AssertParse( + assertParse( """ _ = /x/... - """, - diagnostics: [ - // TODO: Old parser expected error on line 1: unary operator '...' cannot be applied to an operand of type 'Regex' - // TODO: Old parser expected note on line 1: overloads for '...' exist with these partially matching parameter lists - ] + """ ) } func testForwardSlashRegex45() { - AssertParse( + // This is parsed as /x /... + assertParse( """ do { _ = /x1️⃣ /2️⃣... } """, diagnostics: [ - // TODO: Old parser expected error on line 2: '/' is not a prefix unary operator - // TODO: Old parser expected error on line 2: consecutive statements on a line must be separated by ';' // TODO: Old parser expected error on line 2: operator with postfix spacing cannot start a subexpression DiagnosticSpec(locationMarker: "1️⃣", message: "consecutive statements on a line must be separated by ';'"), DiagnosticSpec(locationMarker: "2️⃣", message: "expected expression in prefix operator expression"), DiagnosticSpec(locationMarker: "2️⃣", message: "unexpected code '...' in 'do' statement"), - // TODO: Old parser expected error on line 3: expected expression ] ) } func testForwardSlashRegex46() { - AssertParse( + assertParse( """ do { _ = true / false /1️⃣; @@ -522,7 +400,7 @@ final class ForwardSlashRegexTests: XCTestCase { } func testForwardSlashRegex47() { - AssertParse( + assertParse( #""" _ = "\(/x/)" """# @@ -530,7 +408,7 @@ final class ForwardSlashRegexTests: XCTestCase { } func testForwardSlashRegex48() { - AssertParse( + assertParse( """ func defaulted(x: Regex = /x/) {} """ @@ -538,18 +416,15 @@ final class ForwardSlashRegexTests: XCTestCase { } func testForwardSlashRegex49() { - AssertParse( + assertParse( """ func foo(_ x: T, y: T) {} - """, - diagnostics: [ - // TODO: Old parser expected note on line 1: 'foo(_:y:)' declared here - ] + """ ) } func testForwardSlashRegex50() { - AssertParse( + assertParse( """ foo(/abc/, y: /abc/) """ @@ -557,55 +432,37 @@ final class ForwardSlashRegexTests: XCTestCase { } func testForwardSlashRegex51() { - AssertParse( + assertParse( """ - // TODO: The parser ought to have better recovery in cases where a binary - // operator chain is missing an operand, currently we throw everything away. foo(/abc/, y: /abc /1️⃣) """, diagnostics: [ - // TODO: Old parser expected error on line 3: missing argument for parameter 'y' in call DiagnosticSpec(message: "expected expression in function call"), - // TODO: Old parser expected error on line 3: expected expression after operator ] ) } - func testForwardSlashRegex52() { - AssertParse( - """ - func bar(_ x: inout T) {} - """ - ) - } - func testForwardSlashRegex53() { - AssertParse( + assertParse( """ bar(&/x/) - """, - diagnostics: [ - // TODO: Old parser expected error on line 1: cannot pass immutable value as inout argument: literals are not mutable - ] + """ ) } func testForwardSlashRegex54() { - AssertParse( + assertParse( """ struct S { subscript(x: Regex) -> Void { () } subscript(fn: (Int, Int) -> Int) -> Int { 0 } } - """, - diagnostics: [ - // TODO: Old parser expected note on line 2: 'subscript(_:)' declared here - ] + """ ) } func testForwardSlashRegex55() { - AssertParse( + assertParse( """ func testSubscript(_ x: S) { x[/x/] @@ -614,7 +471,6 @@ final class ForwardSlashRegexTests: XCTestCase { } """, diagnostics: [ - // TODO: Old parser expected error on line 3: missing argument for parameter #1 in call DiagnosticSpec(message: "expected expression in subscript"), // TODO: Old parser expected error on line 3: expected expression after operator ] @@ -622,7 +478,7 @@ final class ForwardSlashRegexTests: XCTestCase { } func testForwardSlashRegex56() { - AssertParse( + assertParse( """ func testReturn() -> Regex { if .random() { @@ -639,20 +495,17 @@ final class ForwardSlashRegexTests: XCTestCase { } func testForwardSlashRegex57() { - AssertParse( + assertParse( """ func testThrow() throws { throw /x/ } - """, - diagnostics: [ - // TODO: Old parser expected error on line 2: thrown expression type 'Regex' does not conform to 'Error' - ] + """ ) } func testForwardSlashRegex58() { - AssertParse( + assertParse( """ do { _ = [/abc/, /abc /1️⃣] @@ -666,7 +519,7 @@ final class ForwardSlashRegexTests: XCTestCase { } func testForwardSlashRegex59() { - AssertParse( + assertParse( """ do { _ = [/abc /1️⃣: /abc /2️⃣] @@ -681,51 +534,39 @@ final class ForwardSlashRegexTests: XCTestCase { } func testForwardSlashRegex60() { - AssertParse( + assertParse( """ _ = [/abc/:/abc/] - """, - diagnostics: [ - // TODO: Old parser expected error on line 1: generic struct 'Dictionary' requires that 'Regex' conform to 'Hashable' - ] + """ ) } func testForwardSlashRegex61() { - AssertParse( + assertParse( """ _ = [/abc/ : /abc/] - """, - diagnostics: [ - // TODO: Old parser expected error on line 1: generic struct 'Dictionary' requires that 'Regex' conform to 'Hashable' - ] + """ ) } func testForwardSlashRegex62() { - AssertParse( + assertParse( """ _ = [/abc/:/abc/] - """, - diagnostics: [ - // TODO: Old parser expected error on line 1: generic struct 'Dictionary' requires that 'Regex' conform to 'Hashable' - ] + """ ) } func testForwardSlashRegex63() { - AssertParse( + assertParse( """ _ = [/abc/: /abc/] - """, - diagnostics: [ - // TODO: Old parser expected error on line 1: generic struct 'Dictionary' requires that 'Regex' conform to 'Hashable' - ] + """ ) } func testForwardSlashRegex64() { - AssertParse( + assertParse( """ _ = (/abc/, /abc/) """ @@ -733,7 +574,7 @@ final class ForwardSlashRegexTests: XCTestCase { } func testForwardSlashRegex65() { - AssertParse( + assertParse( """ _ = ((/abc/)) """ @@ -741,7 +582,7 @@ final class ForwardSlashRegexTests: XCTestCase { } func testForwardSlashRegex66() { - AssertParse( + assertParse( """ do { _ = ((/abc /1️⃣)) @@ -755,7 +596,7 @@ final class ForwardSlashRegexTests: XCTestCase { } func testForwardSlashRegex67() { - AssertParse( + assertParse( """ _ = { /abc/ } """ @@ -763,7 +604,7 @@ final class ForwardSlashRegexTests: XCTestCase { } func testForwardSlashRegex68() { - AssertParse( + assertParse( """ _ = { /abc/ @@ -773,7 +614,7 @@ final class ForwardSlashRegexTests: XCTestCase { } func testForwardSlashRegex69() { - AssertParse( + assertParse( """ let _: () -> Int = { 0 @@ -785,47 +626,33 @@ final class ForwardSlashRegexTests: XCTestCase { } func testForwardSlashRegex70() { - AssertParse( + // This is parsed as '/1 / 2' + assertParse( """ let _: () -> Int = { 0 /1 / 2 } - """, - diagnostics: [ - // TODO: Old parser expected error on line 3: '/' is not a prefix unary operator - ] + """ ) } func testForwardSlashRegex71() { - AssertParse( + assertParse( """ _ = { 0 /1/ 2 } - """, - diagnostics: [ - // TODO: Old parser expected warning on line 2: integer literal is unused - // TODO: Old parser expected warning on line 3: regular expression literal is unused - // TODO: Old parser expected warning on line 4: integer literal is unused - ] - ) - } - - func testForwardSlashRegex72() { - AssertParse( - """ - // Operator chain, as a regex literal may not start with space. """ ) } func testForwardSlashRegex73() { - AssertParse( + // Operator chain, as a regex literal may not start with space. + assertParse( """ _ = 2 / 1 / .bitWidth @@ -834,19 +661,18 @@ final class ForwardSlashRegexTests: XCTestCase { } func testForwardSlashRegex74() { - AssertParse( + // Regex literal + assertParse( """ _ = 2 /1/ .bitWidth - """, - diagnostics: [ - // TODO: Old parser expected error on line 2: value of type 'Regex' has no member 'bitWidth' - ] + """ ) } func testForwardSlashRegex75() { - AssertParse( + // Operator chain + assertParse( """ _ = 2 / 1 / @@ -856,42 +682,36 @@ final class ForwardSlashRegexTests: XCTestCase { } func testForwardSlashRegex76() { - AssertParse( + // This is parsed as '/1 / .bitWidth' + assertParse( """ _ = 2 /1 / .bitWidth - """, - diagnostics: [ - // TODO: Old parser expected error on line 2: '/' is not a prefix unary operator - ] + """ ) } func testForwardSlashRegex77() { - AssertParse( + // This is parsed as /1/.bitWidth + assertParse( """ _ = !!/1/ .bitWidth - """, - diagnostics: [ - // TODO: Old parser expected error on line 1: value of type 'Regex' has no member 'bitWidth' - ] + """ ) } func testForwardSlashRegex78() { - AssertParse( + // This is parsed as '!!/1' + assertParse( """ _ = !!/1 / .bitWidth - """, - diagnostics: [ - // TODO: Old parser expected error on line 1: cannot find operator '!!/' in scope - ] + """ ) } func testForwardSlashRegex79() { - AssertParse( + assertParse( """ let z = /y/ @@ -899,17 +719,10 @@ final class ForwardSlashRegexTests: XCTestCase { ) } - func testForwardSlashRegex80() { - AssertParse( - """ - // While '.' is technically an operator character, it seems more likely that - // the user hasn't written the member name yet. - """ - ) - } - func testForwardSlashRegex81() { - AssertParse( + // While '.' is technically an operator character, it seems more likely that + // the user hasn't written the member name yet. + assertParse( """ _ = 01️⃣. / 1 / 2 """, @@ -921,7 +734,7 @@ final class ForwardSlashRegexTests: XCTestCase { } func testForwardSlashRegex82() { - AssertParse( + assertParse( """ _ = 0 . 1️⃣/ 1 / 2 """, @@ -933,7 +746,7 @@ final class ForwardSlashRegexTests: XCTestCase { } func testForwardSlashRegex83() { - AssertParse( + assertParse( #""" switch "" { case _ where /x/: @@ -941,36 +754,20 @@ final class ForwardSlashRegexTests: XCTestCase { default: break } - """#, - diagnostics: [ - // TODO: Old parser expected error on line 2: cannot convert value of type 'Regex' to expected condition type 'Bool' - ] + """# ) } func testForwardSlashRegex84() { - AssertParse( + assertParse( """ do {} catch /x/ {} """ ) } - func testForwardSlashRegex85() { - AssertParse( - """ - - """, - diagnostics: [ - // TODO: Old parser expected error on line 0: expression pattern of type 'Regex' cannot match values of type 'any Error' - // TODO: Old parser expected error on line 0: binary operator '~=' cannot be applied to two 'any Error' operands - // TODO: Old parser expected warning on line 0: 'catch' block is unreachable because no errors are thrown in 'do' block - ] - ) - } - func testForwardSlashRegex86() { - AssertParse( + assertParse( """ switch /x/ { default: @@ -981,61 +778,41 @@ final class ForwardSlashRegexTests: XCTestCase { } func testForwardSlashRegex87() { - AssertParse( + assertParse( """ if /x/ {} - """, - diagnostics: [ - // TODO: Old parser expected error on line 1: cannot convert value of type 'Regex' to expected condition type 'Bool' - ] + """ ) } func testForwardSlashRegex88() { - AssertParse( + assertParse( """ if /x/.smth {} - """, - diagnostics: [ - // TODO: Old parser expected error on line 1: value of type 'Regex' has no member 'smth' - ] + """ ) } func testForwardSlashRegex89() { - AssertParse( + assertParse( """ func testGuard() { guard /x/ else { return } } - """, - diagnostics: [ - // TODO: Old parser expected error on line 2: cannot convert value of type 'Regex' to expected condition type 'Bool' - ] + """ ) } func testForwardSlashRegex90() { - AssertParse( + assertParse( """ for x in [0] where /x/ {} - """, - diagnostics: [ - // TODO: Old parser expected error on line 1: cannot convert value of type 'Regex' to expected condition type 'Bool' - ] - ) - } - - func testForwardSlashRegex91() { - AssertParse( - """ - typealias Magic = T """ ) } func testForwardSlashRegex92() { - AssertParse( + assertParse( """ _ = /x/ as Magic """ @@ -1043,18 +820,15 @@ final class ForwardSlashRegexTests: XCTestCase { } func testForwardSlashRegex93() { - AssertParse( + assertParse( """ _ = /x/ as! String - """, - diagnostics: [ - // TODO: Old parser expected warning on line 1: cast from 'Regex' to unrelated type 'String' always fails - ] + """ ) } func testForwardSlashRegex94() { - AssertParse( + assertParse( """ _ = type(of: /x/) """ @@ -1062,28 +836,26 @@ final class ForwardSlashRegexTests: XCTestCase { } func testForwardSlashRegex95() { - AssertParse( + assertParse( """ do { let 1️⃣/x/ } """, diagnostics: [ - // TODO: Old parser expected error on line 2: expected pattern DiagnosticSpec(message: "expected pattern in variable"), ] ) } func testForwardSlashRegex96() { - AssertParse( + assertParse( """ do { _ = try /x/; _ = try /x /1️⃣ } """, diagnostics: [ - // TODO: Old parser expected warning on line 2: no calls to throwing functions occur within 'try' expression DiagnosticSpec(message: "expected expression in 'do' statement"), // TODO: Old parser expected error on line 3: expected expression after operator ] @@ -1091,14 +863,13 @@ final class ForwardSlashRegexTests: XCTestCase { } func testForwardSlashRegex97() { - AssertParse( + assertParse( """ do { _ = try? /x/; _ = try? /x /1️⃣ } """, diagnostics: [ - // TODO: Old parser expected warning on line 2: no calls to throwing functions occur within 'try' expression DiagnosticSpec(message: "expected expression in 'do' statement"), // TODO: Old parser expected error on line 3: expected expression after operator ] @@ -1106,14 +877,13 @@ final class ForwardSlashRegexTests: XCTestCase { } func testForwardSlashRegex98() { - AssertParse( + assertParse( """ do { _ = try! /x/; _ = try! /x /1️⃣ } """, diagnostics: [ - // TODO: Old parser expected warning on line 2: no calls to throwing functions occur within 'try' expression DiagnosticSpec(message: "expected expression in 'do' statement"), // TODO: Old parser expected error on line 3: expected expression after operator ] @@ -1121,111 +891,61 @@ final class ForwardSlashRegexTests: XCTestCase { } func testForwardSlashRegex99() { - AssertParse( + assertParse( """ _ = await /x/ - """, - diagnostics: [ - // TODO: Old parser expected warning on line 1: no 'async' operations occur within 'await' expression - ] + """ ) } func testForwardSlashRegex100() { - AssertParse( + assertParse( """ /x/ = 0 /x/() - """, - diagnostics: [ - // TODO: Old parser expected error on line 1: cannot assign to value: literals are not mutable - // TODO: Old parser expected error on line 2: cannot call value of non-function type 'Regex' - ] - ) - } - - func testForwardSlashRegex101() { - AssertParse( - """ - // We treat the following as comments, as it seems more likely the user has - // written a comment and is still in the middle of writing the characters before - // it. """ ) } func testForwardSlashRegex102() { - AssertParse( + // We treat the following as comments, as it seems more likely the user has + // written a comment and is still in the middle of writing the characters before + // it. + assertParse( """ _ = /x// comment - """, - diagnostics: [ - // TODO: Old parser expected error on line 1: '/' is not a prefix unary operator - ] + """ ) } func testForwardSlashRegex103() { - AssertParse( + assertParse( """ _ = /x // comment - """, - diagnostics: [ - // TODO: Old parser expected error on line 1: '/' is not a prefix unary operator - ] + """ ) } func testForwardSlashRegex104() { - AssertParse( + assertParse( """ _ = /x/*comment*/ - """, - diagnostics: [ - // TODO: Old parser expected error on line 1: '/' is not a prefix unary operator - ] - ) - } - - func testForwardSlashRegex105() { - AssertParse( - """ - // MARK: Unapplied operators - """ - ) - } - - func testForwardSlashRegex106() { - AssertParse( - """ - // These become regex literals, unless last character is space, or are surrounded in parens. - """ - ) - } - - func testForwardSlashRegex107() { - AssertParse( - """ - func baz(_ x: (Int, Int) -> Int, _ y: (Int, Int) -> Int) {} // expected-note 3{{'baz' declared here}} """ ) } func testForwardSlashRegex108() { - AssertParse( + // These become regex literals, unless last character is space, or are surrounded in parens. + assertParse( """ baz(/, /) baz(/,/) - """, - diagnostics: [ - // TODO: Old parser expected error on line 2: cannot convert value of type 'Regex' to expected argument type '(Int, Int) -> Int' - // TODO: Old parser expected error on line 2: missing argument for parameter #2 in call - ] + """ ) } func testForwardSlashRegex109() { - AssertParse( + assertParse( """ baz((/), /) """ @@ -1233,20 +953,16 @@ final class ForwardSlashRegexTests: XCTestCase { } func testForwardSlashRegex110() { - AssertParse( + assertParse( """ baz(/^, /) baz(/^,/) - """, - diagnostics: [ - // TODO: Old parser expected error on line 2: cannot convert value of type 'Regex' to expected argument type '(Int, Int) -> Int' - // TODO: Old parser expected error on line 2: missing argument for parameter #2 in call - ] + """ ) } func testForwardSlashRegex111() { - AssertParse( + assertParse( """ baz((/^), /) """ @@ -1254,28 +970,17 @@ final class ForwardSlashRegexTests: XCTestCase { } func testForwardSlashRegex112() { - AssertParse( + assertParse( """ baz(^^/, /) baz(^^/,/) baz((^^/), /) - """, - diagnostics: [ - // TODO: Old parser expected error on line 2: missing argument for parameter #2 in call - ] - ) - } - - func testForwardSlashRegex113() { - AssertParse( - """ - func bazbaz(_ x: (Int, Int) -> Int, _ y: Int) {} """ ) } func testForwardSlashRegex114() { - AssertParse( + assertParse( """ bazbaz(/, 0) bazbaz(^^/, 0) @@ -1283,16 +988,8 @@ final class ForwardSlashRegexTests: XCTestCase { ) } - func testForwardSlashRegex115() { - AssertParse( - """ - func qux(_ x: (Int, Int) -> Int, _ y: T) -> Int { 0 } - """ - ) - } - func testForwardSlashRegex116() { - AssertParse( + assertParse( #""" _ = qux(/, 1) / 2 do { @@ -1301,15 +998,13 @@ final class ForwardSlashRegexTests: XCTestCase { } """#, diagnostics: [ - // TODO: Old parser expected error on line 4: cannot convert value of type 'Regex<(Substring, Substring)>' to expected argument type '(Int, Int) -> Int' DiagnosticSpec(message: "expected ')' to end function call"), - // TODO: Old parser expected error on line 4: expected ',' separator ] ) } func testForwardSlashRegex117() { - AssertParse( + assertParse( #""" _ = qux((/), "(") / 2 """# @@ -1317,7 +1012,7 @@ final class ForwardSlashRegexTests: XCTestCase { } func testForwardSlashRegex118() { - AssertParse( + assertParse( """ _ = qux(/, 1) // this comment tests to make sure we don't try and end the regex on the starting '/' of '//'. """ @@ -1325,60 +1020,33 @@ final class ForwardSlashRegexTests: XCTestCase { } func testForwardSlashRegex119() { - AssertParse( + assertParse( """ _ = qux(/, 1) /* same thing with a block comment */ """ ) } - func testForwardSlashRegex120() { - AssertParse( - """ - @discardableResult1️⃣ - """, - diagnostics: [ - DiagnosticSpec(message: "expected declaration after attribute"), - ] - ) - } - - func testForwardSlashRegex121() { - AssertParse( - """ - func quxqux(_ x: (Int, Int) -> Int) -> Int { 0 } - """ - ) - } - func testForwardSlashRegex122() { - AssertParse( + assertParse( """ quxqux(/^/) quxqux((/^/)) quxqux({ $0 /^/ $1 }) - """, - diagnostics: [ - // TODO: Old parser expected error on line 1: cannot convert value of type 'Regex' to expected argument type '(Int, Int) -> Int' - // TODO: Old parser expected error on line 2: cannot convert value of type 'Regex' to expected argument type '(Int, Int) -> Int' - ] + """ ) } func testForwardSlashRegex123() { - AssertParse( + assertParse( """ quxqux(!/^/) - """, - diagnostics: [ - // TODO: Old parser expected error on line 1: cannot convert value of type 'Bool' to expected argument type '(Int, Int) -> Int' - // TODO: Old parser expected error on line 1: cannot convert value of type 'Regex' to expected argument type 'Bool' - ] + """ ) } func testForwardSlashRegex124() { - AssertParse( + assertParse( """ quxqux(/^) """ @@ -1386,23 +1054,15 @@ final class ForwardSlashRegexTests: XCTestCase { } func testForwardSlashRegex125() { - AssertParse( + assertParse( """ _ = quxqux(/^) / 1 """ ) } - func testForwardSlashRegex126() { - AssertParse( - """ - let arr: [Double] = [2, 3, 4] - """ - ) - } - func testForwardSlashRegex127() { - AssertParse( + assertParse( """ _ = arr.reduce(1, /) / 3 """ @@ -1410,58 +1070,39 @@ final class ForwardSlashRegexTests: XCTestCase { } func testForwardSlashRegex128() { - AssertParse( + assertParse( """ _ = arr.reduce(1, /) + arr.reduce(1, /) """ ) } - func testForwardSlashRegex129() { - AssertParse( - """ - // MARK: ')' disambiguation behavior - """ - ) - } - func testForwardSlashRegex130() { - AssertParse( + assertParse( """ _ = (/x) - """, - diagnostics: [ - // TODO: Old parser expected error on line 1: '/' is not a prefix unary operator - ] + """ ) } func testForwardSlashRegex131() { - AssertParse( + assertParse( """ _ = (/x)/ - """, - diagnostics: [ - // TODO: Old parser expected error on line 1: '/' is not a prefix unary operator - // TODO: Old parser expected error on line 1: '/' is not a postfix unary operator - ] + """ ) } func testForwardSlashRegex132() { - AssertParse( + assertParse( """ _ = (/[(0)])/ - """, - diagnostics: [ - // TODO: Old parser expected error on line 1: '/' is not a prefix unary operator - // TODO: Old parser expected error on line 1: '/' is not a postfix unary operator - ] + """ ) } func testForwardSlashRegex133() { - AssertParse( + assertParse( """ _ = /[(0)]/ """ @@ -1469,7 +1110,7 @@ final class ForwardSlashRegexTests: XCTestCase { } func testForwardSlashRegex134() { - AssertParse( + assertParse( """ _ = /(x)/ """ @@ -1477,55 +1118,39 @@ final class ForwardSlashRegexTests: XCTestCase { } func testForwardSlashRegex135() { - AssertParse( + assertParse( + """ + _ = /[)]/ """ - _ = /[1️⃣)]/ - """, - diagnostics: [ - DiagnosticSpec(message: "expected ']' to end array"), - DiagnosticSpec(message: "extraneous code ')]/' at top level"), - ] ) } func testForwardSlashRegex136() { - AssertParse( + assertParse( #""" - _ = /[a\1️⃣]2️⃣)]/ - """#, - diagnostics: [ - DiagnosticSpec(locationMarker: "1️⃣", message: "expected root in key path"), - DiagnosticSpec(locationMarker: "2️⃣", message: "extraneous code ')]/' at top level"), - ] + _ = /[a\])]/ + """# ) } func testForwardSlashRegex137() { - AssertParse( + assertParse( + """ + _ = /([)])/ """ - _ = /([1️⃣)2️⃣])/ - """, - diagnostics: [ - DiagnosticSpec(locationMarker: "1️⃣", message: "expected ']' to end array"), - DiagnosticSpec(locationMarker: "2️⃣", message: "extraneous code '])/' at top level"), - ] ) } func testForwardSlashRegex138() { - AssertParse( + assertParse( + """ + _ = /]]][)]/ """ - _ = 1️⃣/]]][)]/ - """, - diagnostics: [ - DiagnosticSpec(message: "expected expression"), - DiagnosticSpec(message: "extraneous code '/]]][)]/' at top level"), - ] ) } func testForwardSlashRegex139() { - AssertParse( + assertParse( """ _ = 1️⃣/ """, @@ -1538,13 +1163,12 @@ final class ForwardSlashRegexTests: XCTestCase { } func testForwardSlashRegex140() { - AssertParse( + assertParse( """ _ = 1️⃣/) """, diagnostics: [ // TODO: Old parser expected error on line 1: unterminated regex literal - // TODO: Old parser expected error on line 1: closing ')' does not balance any groups openings DiagnosticSpec(message: "expected expression"), DiagnosticSpec(message: "extraneous code '/)' at top level"), ] @@ -1552,7 +1176,7 @@ final class ForwardSlashRegexTests: XCTestCase { } func testForwardSlashRegex141() { - AssertParse( + assertParse( """ let fn: (Int, Int) -> Int = (/) """ @@ -1560,13 +1184,11 @@ final class ForwardSlashRegexTests: XCTestCase { } func testForwardSlashRegex142() { - AssertParse( + assertParse( #""" _ = /\()1️⃣/ """#, diagnostics: [ - // TODO: Old parser expected error on line 1: '/' is not a prefix unary operator - // TODO: Old parser expected error on line 1: '/' is not a postfix unary operator // TODO: Old parser expected error on line 1: invalid component of Swift key path DiagnosticSpec(message: "extraneous code '/' at top level"), ] @@ -1574,7 +1196,7 @@ final class ForwardSlashRegexTests: XCTestCase { } func testForwardSlashRegex143() { - AssertParse( + assertParse( #""" do { let _: Regex = (/whatever\)/1️⃣ @@ -1583,37 +1205,33 @@ final class ForwardSlashRegexTests: XCTestCase { diagnostics: [ // TODO: Old parser expected note on line 2: to match this opening '(' DiagnosticSpec(message: "expected ')' to end tuple"), - // TODO: Old parser expected error on line 3: expected ')' in expression list ] ) } func testForwardSlashRegex144() { - AssertParse( + assertParse( """ do { _ = /(()())1️⃣)/ } """, diagnostics: [ - // TODO: Old parser expected error on line 2: '/' is not a prefix unary operator // TODO: Old parser expected error on line 2: consecutive statements on a line must be separated by ';' // TODO: Old parser expected error on line 2: expected expression - // TODO: Old parser expected error on line 2: cannot call value of non-function type '()' DiagnosticSpec(message: "unexpected code ')/' in 'do' statement"), ] ) } func testForwardSlashRegex145() { - AssertParse( + assertParse( """ do { _ = /[x]1️⃣)/ } """, diagnostics: [ - // TODO: Old parser expected error on line 2: '/' is not a prefix unary operator // TODO: Old parser expected error on line 2: consecutive statements on a line must be separated by ';' // TODO: Old parser expected error on line 2: expected expression DiagnosticSpec(message: "unexpected code ')/' in 'do' statement"), @@ -1622,7 +1240,7 @@ final class ForwardSlashRegexTests: XCTestCase { } func testForwardSlashRegex146() { - AssertParse( + assertParse( #""" do { _ = /[\1️⃣]2️⃣])/ @@ -1637,29 +1255,21 @@ final class ForwardSlashRegexTests: XCTestCase { } func testForwardSlashRegex147() { - AssertParse( + assertParse( """ _ = ^/x/ - """, - diagnostics: [ - // TODO: Old parser expected error on line 1: '^' is not a prefix unary operator - ] - ) + """) } func testForwardSlashRegex148() { - AssertParse( + assertParse( """ _ = (^/x)/ - """, - diagnostics: [ - // TODO: Old parser expected error on line 1: '/' is not a postfix unary operator - ] - ) + """) } func testForwardSlashRegex149() { - AssertParse( + assertParse( """ _ = (!!/x/) """ @@ -1667,84 +1277,53 @@ final class ForwardSlashRegexTests: XCTestCase { } func testForwardSlashRegex150() { - AssertParse( + assertParse( #""" _ = ^/"/" """#, diagnostics: [ - // TODO: Old parser expected error on line 1: '^' is not a prefix unary operator // TODO: Old parser expected error on line 1: unterminated string literal ] ) } func testForwardSlashRegex151() { - AssertParse( + assertParse( #""" _ = ^/"[/" """#, diagnostics: [ - // TODO: Old parser expected error on line 1: '^' is not a prefix unary operator // TODO: Old parser expected error on line 1: unterminated string literal - // TODO: Old parser expected error on line 1: expected custom character class members ] ) } func testForwardSlashRegex152() { - AssertParse( + assertParse( #""" _ = (^/)("/") """# ) } - func testForwardSlashRegex153() { - AssertParse( - """ - // MARK: Starting characters - """ - ) - } - - func testForwardSlashRegex154() { - AssertParse( - """ - // Fine. - """ - ) - } - func testForwardSlashRegex155() { - AssertParse( + assertParse( """ _ = /./ """ ) } - func testForwardSlashRegex156() { - AssertParse( - """ - // You need to escape if you want a regex literal to start with these characters. - """ - ) - } - func testForwardSlashRegex157() { - AssertParse( + assertParse( #""" - _ = /\ 1️⃣/2️⃣ - """#, - diagnostics: [ - DiagnosticSpec(locationMarker: "1️⃣", message: "expected root in key path"), - DiagnosticSpec(locationMarker: "2️⃣", message: "expected expression"), - ] + _ = /\ / + """# ) } func testForwardSlashRegex158() { - AssertParse( + assertParse( """ _ = 1️⃣/ / """, @@ -1757,7 +1336,7 @@ final class ForwardSlashRegexTests: XCTestCase { } func testForwardSlashRegex159() { - AssertParse( + assertParse( """ _ = 1️⃣/ / """, @@ -1771,7 +1350,7 @@ final class ForwardSlashRegexTests: XCTestCase { } func testForwardSlashRegex160() { - AssertParse( + assertParse( """ _ = #/ /# """ @@ -1779,43 +1358,33 @@ final class ForwardSlashRegexTests: XCTestCase { } func testForwardSlashRegex161() { - AssertParse( + assertParse( #""" - _ = /x1️⃣\ 2️⃣/3️⃣ - """#, - diagnostics: [ - DiagnosticSpec(locationMarker: "1️⃣", message: "consecutive statements on a line must be separated by ';'"), - DiagnosticSpec(locationMarker: "2️⃣", message: "expected root in key path"), - DiagnosticSpec(locationMarker: "3️⃣", message: "expected expression"), - ] + _ = /x\ / + """# ) } func testForwardSlashRegex162() { - AssertParse( + assertParse( #""" - _ = /\ 1️⃣\ 2️⃣/3️⃣ - """#, - diagnostics: [ - DiagnosticSpec(locationMarker: "1️⃣", message: "expected root in key path"), - DiagnosticSpec(locationMarker: "2️⃣", message: "expected root in key path"), - DiagnosticSpec(locationMarker: "3️⃣", message: "expected expression"), - ] + _ = /\ \ / + """# ) } func testForwardSlashRegex163() { - AssertParse( + assertParse( """ - // There are intentionally trailing spaces here """ ) } func testForwardSlashRegex164() { - AssertParse( + // There are intentionally trailing spaces here + assertParse( """ - _ = 1️⃣/ + _ = 1️⃣/ """, diagnostics: [ // TODO: Old parser expected error on line 1: unterminated regex literal @@ -1826,18 +1395,11 @@ final class ForwardSlashRegexTests: XCTestCase { ) } - func testForwardSlashRegex165() { - AssertParse( - """ - // There are intentionally trailing spaces here - """ - ) - } - func testForwardSlashRegex166() { - AssertParse( + // There are intentionally trailing spaces here + assertParse( """ - _ = 1️⃣/^ + _ = 1️⃣/^ """, diagnostics: [ // TODO: Old parser expected error on line 1: unterminated regex literal @@ -1848,7 +1410,7 @@ final class ForwardSlashRegexTests: XCTestCase { } func testForwardSlashRegex167() { - AssertParse( + assertParse( #""" _ = /\)/ """# @@ -1856,12 +1418,11 @@ final class ForwardSlashRegexTests: XCTestCase { } func testForwardSlashRegex168() { - AssertParse( + assertParse( """ _ = 1️⃣/)/ """, diagnostics: [ - // TODO: Old parser expected error on line 1: closing ')' does not balance any groups openings DiagnosticSpec(message: "expected expression"), DiagnosticSpec(message: "extraneous code '/)/' at top level"), ] @@ -1869,7 +1430,7 @@ final class ForwardSlashRegexTests: XCTestCase { } func testForwardSlashRegex169() { - AssertParse( + assertParse( """ _ = /,/ """ @@ -1877,7 +1438,7 @@ final class ForwardSlashRegexTests: XCTestCase { } func testForwardSlashRegex170() { - AssertParse( + assertParse( """ _ = /}/ """ @@ -1885,7 +1446,7 @@ final class ForwardSlashRegexTests: XCTestCase { } func testForwardSlashRegex171() { - AssertParse( + assertParse( """ _ = /]/ """ @@ -1893,7 +1454,7 @@ final class ForwardSlashRegexTests: XCTestCase { } func testForwardSlashRegex172() { - AssertParse( + assertParse( """ _ = /:/ """ @@ -1901,23 +1462,15 @@ final class ForwardSlashRegexTests: XCTestCase { } func testForwardSlashRegex173() { - AssertParse( + assertParse( """ _ = /;/ """ ) } - func testForwardSlashRegex174() { - AssertParse( - """ - // Don't emit diagnostics here, as we re-lex. - """ - ) - } - func testForwardSlashRegex175() { - AssertParse( + assertParse( """ _ = /0xG/ """ @@ -1925,7 +1478,7 @@ final class ForwardSlashRegexTests: XCTestCase { } func testForwardSlashRegex176() { - AssertParse( + assertParse( """ _ = /0oG/ """ @@ -1933,7 +1486,7 @@ final class ForwardSlashRegexTests: XCTestCase { } func testForwardSlashRegex177() { - AssertParse( + assertParse( #""" _ = /"/ """# @@ -1941,7 +1494,7 @@ final class ForwardSlashRegexTests: XCTestCase { } func testForwardSlashRegex178() { - AssertParse( + assertParse( """ _ = /'/ """ @@ -1949,7 +1502,7 @@ final class ForwardSlashRegexTests: XCTestCase { } func testForwardSlashRegex179() { - AssertParse( + assertParse( """ _ = /<#placeholder#>/ """ @@ -1957,11 +1510,12 @@ final class ForwardSlashRegexTests: XCTestCase { } func testForwardSlashRegex180() { - AssertParse( + assertParse( """ _ = ^^/1️⃣0xG/ """, diagnostics: [ + // FIXME: This should not error DiagnosticSpec(message: "expected expression in prefix operator expression"), DiagnosticSpec(message: "extraneous code '0xG/' at top level"), ] @@ -1969,11 +1523,12 @@ final class ForwardSlashRegexTests: XCTestCase { } func testForwardSlashRegex181() { - AssertParse( + assertParse( """ _ = ^^/1️⃣0oG/ """, diagnostics: [ + // FIXME: This should not error DiagnosticSpec(message: "expected expression in prefix operator expression"), DiagnosticSpec(message: "extraneous code '0oG/' at top level"), ] @@ -1981,29 +1536,31 @@ final class ForwardSlashRegexTests: XCTestCase { } func testForwardSlashRegex182() { - AssertParse( + assertParse( #""" _ = ^^/"/1️⃣ """#, diagnostics: [ + // FIXME: This should not error DiagnosticSpec(message: #"expected '"' to end string literal"#), ] ) } func testForwardSlashRegex183() { - AssertParse( + assertParse( """ _ = ^^/'/1️⃣ """, diagnostics: [ + // FIXME: This should not error DiagnosticSpec(message: "expected ''' in string literal"), ] ) } func testForwardSlashRegex184() { - AssertParse( + assertParse( """ _ = ^^/<#placeholder#>/ """ @@ -2011,11 +1568,12 @@ final class ForwardSlashRegexTests: XCTestCase { } func testForwardSlashRegex185() { - AssertParse( + assertParse( """ _ = (^^/1️⃣0xG/) """, diagnostics: [ + // FIXME: This should not error DiagnosticSpec(message: "expected expression in prefix operator expression"), DiagnosticSpec(message: "unexpected code '0xG/' in tuple"), ] @@ -2023,11 +1581,12 @@ final class ForwardSlashRegexTests: XCTestCase { } func testForwardSlashRegex186() { - AssertParse( + assertParse( """ _ = (^^/1️⃣0oG/) """, diagnostics: [ + // FIXME: This should not error DiagnosticSpec(message: "expected expression in prefix operator expression"), DiagnosticSpec(message: "unexpected code '0oG/' in tuple"), ] @@ -2035,11 +1594,12 @@ final class ForwardSlashRegexTests: XCTestCase { } func testForwardSlashRegex187() { - AssertParse( + assertParse( #""" _ = (^^/"/)1️⃣ """#, diagnostics: [ + // FIXME: This should not error DiagnosticSpec(message: #"expected '"' to end string literal"#), DiagnosticSpec(message: "expected ')' to end tuple"), ] @@ -2047,11 +1607,12 @@ final class ForwardSlashRegexTests: XCTestCase { } func testForwardSlashRegex188() { - AssertParse( + assertParse( """ _ = (^^/'/)1️⃣ """, diagnostics: [ + // FIXME: This should not error DiagnosticSpec(message: "expected ''' in string literal"), DiagnosticSpec(message: "expected ')' to end tuple"), ] @@ -2059,7 +1620,7 @@ final class ForwardSlashRegexTests: XCTestCase { } func testForwardSlashRegex189() { - AssertParse( + assertParse( """ _ = (^^/<#placeholder#>/) """ diff --git a/Tests/SwiftParserTest/translated/PrefixSlashTests.swift b/Tests/SwiftParserTest/translated/PrefixSlashTests.swift index c9b23853fcd..9e331dfedfe 100644 --- a/Tests/SwiftParserTest/translated/PrefixSlashTests.swift +++ b/Tests/SwiftParserTest/translated/PrefixSlashTests.swift @@ -4,7 +4,7 @@ import XCTest final class PrefixSlashTests: XCTestCase { func testPrefixSlash1() { - AssertParse( + assertParse( """ // Test the behavior of prefix '/' with regex literals enabled. """ @@ -12,7 +12,7 @@ final class PrefixSlashTests: XCTestCase { } func testPrefixSlash2() { - AssertParse( + assertParse( """ prefix operator / prefix func / (_ x: T) -> T { x } @@ -21,7 +21,7 @@ final class PrefixSlashTests: XCTestCase { } func testPrefixSlash3() { - AssertParse( + assertParse( """ enum E { case e @@ -32,7 +32,7 @@ final class PrefixSlashTests: XCTestCase { } func testPrefixSlash4() { - AssertParse( + assertParse( """ _ = /E.e (/E.e).foo(/0) @@ -41,7 +41,7 @@ final class PrefixSlashTests: XCTestCase { } func testPrefixSlash5() { - AssertParse( + assertParse( """ func foo(_ x: T, _ y: U) {} """ @@ -49,7 +49,7 @@ final class PrefixSlashTests: XCTestCase { } func testPrefixSlash6() { - AssertParse( + assertParse( """ foo(/E.e, /E.e) foo((/E.e), /E.e) @@ -59,7 +59,7 @@ final class PrefixSlashTests: XCTestCase { } func testPrefixSlash7() { - AssertParse( + assertParse( """ func bar(_ x: T) -> Int { 0 } """ @@ -67,7 +67,7 @@ final class PrefixSlashTests: XCTestCase { } func testPrefixSlash8() { - AssertParse( + assertParse( """ _ = bar(/E.e) / 2 """ diff --git a/Tests/SwiftParserTest/translated/RegexParseEndOfBufferTests.swift b/Tests/SwiftParserTest/translated/RegexParseEndOfBufferTests.swift index 49d3bd1f993..ea500fe71f8 100644 --- a/Tests/SwiftParserTest/translated/RegexParseEndOfBufferTests.swift +++ b/Tests/SwiftParserTest/translated/RegexParseEndOfBufferTests.swift @@ -4,16 +4,13 @@ import XCTest final class RegexParseEndOfBufferTests: XCTestCase { func testRegexParseEndOfBuffer1() { - AssertParse( + assertParse( """ // Note there is purposefully no trailing newline here. - var unterminated = 1️⃣#/(xy2️⃣ + var unterminated = 1️⃣#/(xy """, diagnostics: [ - DiagnosticSpec(locationMarker: "1️⃣", message: "use of unknown directive '#'"), - // TODO: Old parser expected error on line 2: unterminated regex literal - DiagnosticSpec(locationMarker: "2️⃣", message: "expected ')' to end tuple"), - // TODO: Old parser expected error on line 2: cannot parse regular expression: expected ')' + DiagnosticSpec(message: "unterminated regex literal") ] ) } diff --git a/Tests/SwiftParserTest/translated/RegexParseErrorTests.swift b/Tests/SwiftParserTest/translated/RegexParseErrorTests.swift index 98427f759ed..22d504f1c7b 100644 --- a/Tests/SwiftParserTest/translated/RegexParseErrorTests.swift +++ b/Tests/SwiftParserTest/translated/RegexParseErrorTests.swift @@ -4,37 +4,23 @@ import XCTest final class RegexParseErrorTests: XCTestCase { func testRegexParseError1() { - AssertParse( + assertParse( """ _ = /(/ - """, - diagnostics: [ - // TODO: Old parser expected error on line 1: expected ')' - ] + """ ) } func testRegexParseError2() { - AssertParse( + assertParse( """ _ = #/(/# - """, - diagnostics: [ - // TODO: Old parser expected error on line 1: expected ')' - ] - ) - } - - func testRegexParseError3() { - AssertParse( - """ - // FIXME: Should be 'group openings' """ ) } func testRegexParseError4() { - AssertParse( + assertParse( """ _ = 1️⃣/)/ """, @@ -47,135 +33,104 @@ final class RegexParseErrorTests: XCTestCase { } func testRegexParseError5() { - AssertParse( + assertParse( + """ + _ = #/)/# """ - _ = 1️⃣#/2️⃣)/# - """, - diagnostics: [ - DiagnosticSpec(locationMarker: "1️⃣", message: "use of unknown directive '#'"), - DiagnosticSpec(locationMarker: "2️⃣", message: "extraneous code ')/#' at top level"), - // TODO: Old parser expected error on line 1: closing ')' does not balance any groups openings - ] ) } func testRegexParseError6() { - AssertParse( + assertParse( #""" - _ = 1️⃣#/\2️⃣\3️⃣/''/ + _ = 1️⃣#/\\/''/ """#, diagnostics: [ - DiagnosticSpec(locationMarker: "1️⃣", message: "use of unknown directive '#'"), - // TODO: Old parser expected error on line 1: unterminated regex literal - DiagnosticSpec(locationMarker: "2️⃣", message: "expected root in key path"), - DiagnosticSpec(locationMarker: "3️⃣", message: "expected root in key path"), + DiagnosticSpec(message: "unterminated regex literal") ] ) } func testRegexParseError7() { - AssertParse( + assertParse( #""" - _ = 1️⃣#/\2️⃣| + _ = 1️⃣#/\| """#, diagnostics: [ - DiagnosticSpec(locationMarker: "1️⃣", message: "use of unknown directive '#'"), - // TODO: Old parser expected error on line 1: unterminated regex literal - DiagnosticSpec(locationMarker: "2️⃣", message: "expected root in key path"), - DiagnosticSpec(locationMarker: "2️⃣", message: "extraneous code '|' at top level"), + DiagnosticSpec(message: "unterminated regex literal") ] ) } func testRegexParseError8() { - AssertParse( + assertParse( """ _ = 1️⃣#// """, diagnostics: [ - DiagnosticSpec(message: "use of unknown directive '#'"), - // TODO: Old parser expected error on line 1: unterminated regex literal + DiagnosticSpec(message: "unterminated regex literal") ] ) } func testRegexParseError9() { - AssertParse( + assertParse( """ _ = 1️⃣#/xy """, diagnostics: [ - DiagnosticSpec(message: "use of unknown directive '#'"), - // TODO: Old parser expected error on line 1: unterminated regex literal + DiagnosticSpec(message: "unterminated regex literal") ] ) } func testRegexParseError10() { - AssertParse( + assertParse( """ _ = #/(?/# - """, - diagnostics: [ - // TODO: Old parser expected error on line 1: expected group specifier - ] + """ ) } func testRegexParseError11() { - AssertParse( + assertParse( """ _ = #/(?'/# - """, - diagnostics: [ - // TODO: Old parser expected error on line 1: expected group name - ] + """ ) } func testRegexParseError12() { - AssertParse( + assertParse( """ _ = #/(?'abc/# - """, - diagnostics: [ - // TODO: Old parser expected error on line 1: expected ''' - ] + """ ) } func testRegexParseError13() { - AssertParse( + assertParse( """ _ = #/(?'abc /# - """, - diagnostics: [ - // TODO: Old parser expected error on line 1: expected ''' - ] + """ ) } func testRegexParseError14() { - AssertParse( + assertParse( """ do { - _ = 1️⃣#/(2️⃣?3️⃣'a + _ = 1️⃣#/(?'a } """, diagnostics: [ - DiagnosticSpec(locationMarker: "1️⃣", message: "use of unknown directive '#'"), - // TODO: Old parser expected error on line 2: unterminated regex literal - DiagnosticSpec(locationMarker: "2️⃣", message: "expected value and ')' to end tuple"), - DiagnosticSpec(locationMarker: "3️⃣", message: "expected expression and ':' in 'do' statement"), - DiagnosticSpec(locationMarker: "3️⃣", message: "expected expression in 'do' statement"), - DiagnosticSpec(locationMarker: "3️⃣", message: "unexpected code ''a' in 'do' statement"), - // TODO: Old parser expected error on line 2: cannot parse regular expression: expected ''' + DiagnosticSpec(message: "unterminated regex literal") ] ) } func testRegexParseError15() { - AssertParse( + assertParse( #""" _ = #/\(?'abc/# """# @@ -183,7 +138,7 @@ final class RegexParseErrorTests: XCTestCase { } func testRegexParseError16() { - AssertParse( + assertParse( #""" do { _ = /\1️⃣ @@ -199,23 +154,22 @@ final class RegexParseErrorTests: XCTestCase { } func testRegexParseError17() { - AssertParse( + assertParse( #""" do { - _ = #/\ - /# + _ = 1️⃣#/\ + /#2️⃣ } """#, diagnostics: [ - // TODO: Old parser expected error on line 2: unterminated regex literal - // TODO: Old parser expected error on line 2: expected escape sequence - // TODO: Old parser expected error on line 3: expected expression + DiagnosticSpec(locationMarker: "1️⃣", message: "unterminated regex literal"), + DiagnosticSpec(locationMarker: "2️⃣", message: "expected identifier in pound literal expression") ] ) } func testRegexParseError18() { - AssertParse( + assertParse( """ func foo( _ x: T, @@ -225,27 +179,19 @@ final class RegexParseErrorTests: XCTestCase { } func testRegexParseError19() { - AssertParse( + assertParse( """ foo(#/(?/#, #/abc/#) foo(#/(?C/#, #/abc/#) - """, - diagnostics: [ - // TODO: Old parser expected error on line 1: expected group specifier - // TODO: Old parser expected error on line 2: expected ')' - ] + """ ) } func testRegexParseError20() { - AssertParse( + assertParse( """ foo(#/(?'/#, #/abc/#) - """, - diagnostics: [ - // TODO: Old parser expected error on line 1: expected group name - ] + """ ) } - } diff --git a/Tests/SwiftParserTest/translated/RegexTests.swift b/Tests/SwiftParserTest/translated/RegexTests.swift index 102c823a3b7..18465f89099 100644 --- a/Tests/SwiftParserTest/translated/RegexTests.swift +++ b/Tests/SwiftParserTest/translated/RegexTests.swift @@ -4,7 +4,7 @@ import XCTest final class RegexTests: XCTestCase { func testRegex1() { - AssertParse( + assertParse( """ _ = /abc/ _ = #/abc/# @@ -14,7 +14,7 @@ final class RegexTests: XCTestCase { } func testRegex2() { - AssertParse( + assertParse( """ func foo(_ x: T...) {} """ @@ -22,7 +22,7 @@ final class RegexTests: XCTestCase { } func testRegex3() { - AssertParse( + assertParse( """ foo(/abc/, #/abc/#, ##/abc/##) """ @@ -30,7 +30,7 @@ final class RegexTests: XCTestCase { } func testRegex4() { - AssertParse( + assertParse( """ let arr = [/abc/, #/abc/#, ##/abc/##] """ @@ -38,7 +38,7 @@ final class RegexTests: XCTestCase { } func testRegex5() { - AssertParse( + assertParse( #""" _ = /\w+/.self _ = #/\w+/#.self @@ -48,7 +48,7 @@ final class RegexTests: XCTestCase { } func testRegex6() { - AssertParse( + assertParse( ##""" _ = /#\/\#\\/ _ = #/#/\/\#\\/# @@ -58,14 +58,10 @@ final class RegexTests: XCTestCase { } func testRegex7() { - AssertParse( + assertParse( """ _ = (#/[*/#, #/+]/#, #/.]/#) - """, - diagnostics: [ - // TODO: Old parser expected error on line 1: cannot parse regular expression: expected ']' - // TODO: Old parser expected error on line 1: cannot parse regular expression: quantifier '+' must appear after expression - ] + """ ) } From f16c319efea4c2e923d4f07cb27aadfe47f7e91a Mon Sep 17 00:00:00 2001 From: Hamish Knight Date: Wed, 29 Mar 2023 00:25:21 +0100 Subject: [PATCH 03/12] Track the previous keyword in Cursor We currently track the previous raw token kind, but that is not sufficient to identify keyword kinds. Additionally store the keyword kind. --- Sources/SwiftParser/Lexer/Cursor.swift | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/Sources/SwiftParser/Lexer/Cursor.swift b/Sources/SwiftParser/Lexer/Cursor.swift index cc59d28249b..f8880400baa 100644 --- a/Sources/SwiftParser/Lexer/Cursor.swift +++ b/Sources/SwiftParser/Lexer/Cursor.swift @@ -217,6 +217,8 @@ extension Lexer { /// If we have already lexed a token, the kind of the previously lexed token var previousTokenKind: RawTokenKind? + var previousKeyword: Keyword? + private var stateStack: StateStack = StateStack() init(input: UnsafeBufferPointer, previous: UInt8) { @@ -372,10 +374,9 @@ extension Lexer.Cursor { flags.insert(.isAtStartOfLine) } - self.previousTokenKind = result.tokenKind diagnostic = TokenDiagnostic(combining: diagnostic, result.error?.tokenDiagnostic(tokenStart: cursor)) - return .init( + let lexeme = Lexer.Lexeme( tokenKind: result.tokenKind, flags: flags, diagnostic: diagnostic, @@ -385,6 +386,10 @@ extension Lexer.Cursor { trailingTriviaLength: trailingTriviaStart.distance(to: self), cursor: cursor ) + self.previousTokenKind = result.tokenKind + self.previousKeyword = result.tokenKind == .keyword ? Keyword(lexeme.tokenText)! : nil + + return lexeme } } From 3eb4394d202ab1421d9f67e9f1bedfe9ffb481cd Mon Sep 17 00:00:00 2001 From: Hamish Knight Date: Wed, 29 Mar 2023 00:25:22 +0100 Subject: [PATCH 04/12] Remove ForwardSlashRegexDisabledTests.swift The SwiftSyntax parser doesn't have a way of disabling bare slash regex literals, so this is unnecessary. --- .../ForwardSlashRegexDisabledTests.swift | 94 ------------------- 1 file changed, 94 deletions(-) delete mode 100644 Tests/SwiftParserTest/translated/ForwardSlashRegexDisabledTests.swift diff --git a/Tests/SwiftParserTest/translated/ForwardSlashRegexDisabledTests.swift b/Tests/SwiftParserTest/translated/ForwardSlashRegexDisabledTests.swift deleted file mode 100644 index 655ef15b181..00000000000 --- a/Tests/SwiftParserTest/translated/ForwardSlashRegexDisabledTests.swift +++ /dev/null @@ -1,94 +0,0 @@ -// This test file has been translated from swift/test/StringProcessing/Parse/forward-slash-regex-disabled.swift - -import XCTest - -final class ForwardSlashRegexDisabledTests: XCTestCase { - func testForwardSlashRegexDisabled1() { - AssertParse( - """ - prefix operator / - prefix operator ^/ - prefix operator /^/ - """ - ) - } - - func testForwardSlashRegexDisabled2() { - AssertParse( - """ - precedencegroup P { - associativity: left - } - """ - ) - } - - func testForwardSlashRegexDisabled3() { - AssertParse( - """ - // The divisions in the body of the below operators make sure we don't try and - // consider them to be ending delimiters of a regex. - infix operator /^/ : P - func /^/ (lhs: Int, rhs: Int) -> Int { 1 / 2 } - """ - ) - } - - func testForwardSlashRegexDisabled4() { - AssertParse( - """ - infix operator /^ : P - func /^ (lhs: Int, rhs: Int) -> Int { 1 / 2 } - """ - ) - } - - func testForwardSlashRegexDisabled5() { - AssertParse( - """ - infix operator ^^/ : P - func ^^/ (lhs: Int, rhs: Int) -> Int { 1 / 2 } - """ - ) - } - - func testForwardSlashRegexDisabled6() { - AssertParse( - """ - _ = #/x/# - """ - ) - } - - func testForwardSlashRegexDisabled7() { - AssertParse( - """ - _ = /x/ - """, - diagnostics: [ - // TODO: Old parser expected error on line 1: '/' is not a prefix unary operator - // TODO: Old parser expected error on line 1: cannot find 'x' in scope - // TODO: Old parser expected error on line 1: '/' is not a postfix unary operator - ] - ) - } - - func testForwardSlashRegexDisabled8() { - AssertParse( - """ - func baz(_ x: (Int, Int) -> Int, _ y: (Int, Int) -> Int) {} - """ - ) - } - - func testForwardSlashRegexDisabled9() { - AssertParse( - """ - baz(/, /) - baz(/^, /) - baz(^^/, /) - """ - ) - } - -} From 09122a9c4c6644c5cf69d2bb6fc4c1f32d72ae01 Mon Sep 17 00:00:00 2001 From: Hamish Knight Date: Wed, 29 Mar 2023 00:25:23 +0100 Subject: [PATCH 05/12] Refactor operator lexing in `lexNormal` a bit Many of these can fall through to the default case to lex the operator. Also factor out `lexPostfixOptionalChain` in preparation for a bit of custom state transition logic. --- Sources/SwiftParser/Lexer/Cursor.swift | 50 ++++++++++++++------------ 1 file changed, 28 insertions(+), 22 deletions(-) diff --git a/Sources/SwiftParser/Lexer/Cursor.swift b/Sources/SwiftParser/Lexer/Cursor.swift index f8880400baa..2cbbc26a176 100644 --- a/Sources/SwiftParser/Lexer/Cursor.swift +++ b/Sources/SwiftParser/Lexer/Cursor.swift @@ -816,33 +816,20 @@ extension Lexer.Cursor { // Otherwise try lex a magic pound literal. return self.lexOperatorIdentifier(sourceBufferStart: sourceBufferStart) - case UInt8(ascii: "!"): - if self.isLeftBound(sourceBufferStart: sourceBufferStart) { - _ = self.advance() - return Lexer.Result(.exclamationMark) - } - return self.lexOperatorIdentifier(sourceBufferStart: sourceBufferStart) - - case UInt8(ascii: "?"): - if self.isLeftBound(sourceBufferStart: sourceBufferStart) { - _ = self.advance() - return Lexer.Result(.postfixQuestionMark) + case UInt8(ascii: "!"), UInt8(ascii: "?"): + if let result = lexPostfixOptionalChain(sourceBufferStart: sourceBufferStart) { + return result } return self.lexOperatorIdentifier(sourceBufferStart: sourceBufferStart) case UInt8(ascii: "<"): - if self.is(offset: 1, at: "#") { - return self.tryLexEditorPlaceholder(sourceBufferStart: sourceBufferStart) + if self.is(offset: 1, at: "#"), + let result = self.tryLexEditorPlaceholder(sourceBufferStart: sourceBufferStart) + { + return result } return self.lexOperatorIdentifier(sourceBufferStart: sourceBufferStart) - case UInt8(ascii: ">"): - return self.lexOperatorIdentifier(sourceBufferStart: sourceBufferStart) - case UInt8(ascii: "="), UInt8(ascii: "-"), UInt8(ascii: "+"), - UInt8(ascii: "*"), UInt8(ascii: "%"), UInt8(ascii: "&"), - UInt8(ascii: "|"), UInt8(ascii: "^"), UInt8(ascii: "~"), - UInt8(ascii: "."): - return self.lexOperatorIdentifier(sourceBufferStart: sourceBufferStart) case UInt8(ascii: "A"), UInt8(ascii: "B"), UInt8(ascii: "C"), UInt8(ascii: "D"), UInt8(ascii: "E"), UInt8(ascii: "F"), UInt8(ascii: "G"), UInt8(ascii: "H"), UInt8(ascii: "I"), @@ -1924,6 +1911,25 @@ extension Lexer.Cursor { return Lexer.Result(.backtick) } + /// Attempt to lex a postfix '!' or '?'. + mutating func lexPostfixOptionalChain(sourceBufferStart: Lexer.Cursor) -> Lexer.Result? { + // Must be left bound, otherwise this isn't postfix. + guard self.isLeftBound(sourceBufferStart: sourceBufferStart) else { return nil } + + let kind: RawTokenKind = { + switch self.peek() { + case UInt8(ascii: "!"): + return .exclamationMark + case UInt8(ascii: "?"): + return .postfixQuestionMark + default: + fatalError("Must be at '!' or '?'") + } + }() + _ = self.advance() + return Lexer.Result(kind) + } + mutating func lexOperatorIdentifier(sourceBufferStart: Lexer.Cursor) -> Lexer.Result { let tokStart = self let didStart = self.advance(if: { $0.isOperatorStartCodePoint }) @@ -2065,7 +2071,7 @@ extension Lexer.Cursor { // MARK: - Editor Placeholders extension Lexer.Cursor { - mutating func tryLexEditorPlaceholder(sourceBufferStart: Lexer.Cursor) -> Lexer.Result { + mutating func tryLexEditorPlaceholder(sourceBufferStart: Lexer.Cursor) -> Lexer.Result? { precondition(self.is(at: "<") && self.is(offset: 1, at: "#")) let start = self var ptr = self @@ -2092,7 +2098,7 @@ extension Lexer.Cursor { } // Not a well-formed placeholder. - return self.lexOperatorIdentifier(sourceBufferStart: sourceBufferStart) + return nil } } From 75504963ab005b3b59e05fb4a1fd0b7d21a6f1fd Mon Sep 17 00:00:00 2001 From: Hamish Knight Date: Wed, 29 Mar 2023 00:25:23 +0100 Subject: [PATCH 06/12] Generalize raw string closing delimiter parsing logic Generalize in preparation for its use by regex literals. --- Sources/SwiftParser/StringLiterals.swift | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/Sources/SwiftParser/StringLiterals.swift b/Sources/SwiftParser/StringLiterals.swift index 8de1386832d..ffd756e6b49 100644 --- a/Sources/SwiftParser/StringLiterals.swift +++ b/Sources/SwiftParser/StringLiterals.swift @@ -426,8 +426,12 @@ extension Parser { // MARK: - Parse string literals extension Parser { - /// Consumes a raw string delimiter that has the same number of `#` as `openDelimiter`. - private mutating func parseStringDelimiter(openDelimiter: RawTokenSyntax?) -> (unexpectedBeforeCheckedDelimiter: RawUnexpectedNodesSyntax?, checkedDelimiter: RawTokenSyntax?) { + /// Consumes a raw string or extended regex delimiter that has the same + /// number of `#` as `openDelimiter`. + mutating func parsePoundDelimiter( + _ kind: RawTokenKind, + matching openDelimiter: RawTokenSyntax? + ) -> (unexpectedBeforeCheckedDelimiter: RawUnexpectedNodesSyntax?, checkedDelimiter: RawTokenSyntax?) { // Check for leadingTriviaText == "" so we don't consume the leading raw // string delimiter of an upcoming string literal, e.g. in // ``` @@ -435,7 +439,7 @@ extension Parser { // #"raw literal"# // ``` let delimiter: RawTokenSyntax? - if self.at(.rawStringDelimiter) && self.currentToken.leadingTriviaText == "" { + if self.at(TokenSpec(kind)) && self.currentToken.leadingTriviaText == "" { delimiter = self.consumeAnyToken() } else { delimiter = nil @@ -445,14 +449,14 @@ extension Parser { case (nil, nil): return (nil, nil) case (let open?, nil): - return (nil, missingToken(.rawStringDelimiter, text: open.tokenText)) + return (nil, missingToken(kind, text: open.tokenText)) case (nil, .some): return (RawUnexpectedNodesSyntax([delimiter], arena: self.arena), nil) case (let open?, let close?): if open.tokenText == close.tokenText { return (nil, close) } else { - return (RawUnexpectedNodesSyntax([delimiter], arena: self.arena), missingToken(.rawStringDelimiter, text: open.tokenText)) + return (RawUnexpectedNodesSyntax([delimiter], arena: self.arena), missingToken(kind, text: open.tokenText)) } } } @@ -482,7 +486,7 @@ extension Parser { if let stringSegment = self.consume(if: .stringSegment) { segments.append(.stringSegment(RawStringSegmentSyntax(content: stringSegment, arena: self.arena))) } else if let backslash = self.consume(if: .backslash) { - let (unexpectedBeforeDelimiter, delimiter) = self.parseStringDelimiter(openDelimiter: openDelimiter) + let (unexpectedBeforeDelimiter, delimiter) = self.parsePoundDelimiter(.rawStringDelimiter, matching: openDelimiter) let leftParen = self.expectWithoutRecoveryOrLeadingTrivia(.leftParen) let expressions = RawTupleExprElementListSyntax(elements: self.parseArgumentListElements(pattern: .none), arena: self.arena) @@ -546,7 +550,7 @@ extension Parser { closeQuote = self.expectWithoutRecoveryOrLeadingTrivia(TokenSpec(openQuote.tokenKind)) } - let (unexpectedBeforeCloseDelimiter, closeDelimiter) = self.parseStringDelimiter(openDelimiter: openDelimiter) + let (unexpectedBeforeCloseDelimiter, closeDelimiter) = self.parsePoundDelimiter(.rawStringDelimiter, matching: openDelimiter) if openQuote.tokenKind == .multilineStringQuote, !openQuote.isMissing, !closeQuote.isMissing { let postProcessed = postProcessMultilineStringLiteral(rawStringDelimitersToken: openDelimiter, openQuote: openQuote, segments: segments, closeQuote: closeQuote) From 0f80f2b6f2853facb82504777d2758f828353164 Mon Sep 17 00:00:00 2001 From: Hamish Knight Date: Wed, 29 Mar 2023 00:25:23 +0100 Subject: [PATCH 07/12] Complete regex literal lexing The previous regex literal lexing logic didn't implement all the heuristics needed to parse all the regex literals that the old C++ parser handles. Update it with the heuristics it needs, and better handle diagnostics for invalid cases. This unfortunately requires a bit of lexical hackery, including a heuristic to classify previous token kinds to determine if we're in regex literal position, but it's needed to handle pathological cases such as `x /^ y/` where there's an ambiguity with a binary operator. --- .../SyntaxSupport/Classification.swift | 1 + .../Sources/SyntaxSupport/ExprNodes.swift | 24 +- .../Sources/SyntaxSupport/TokenSpec.swift | 4 +- Sources/SwiftParser/CMakeLists.txt | 1 + Sources/SwiftParser/Declarations.swift | 24 +- Sources/SwiftParser/Expressions.swift | 29 +- Sources/SwiftParser/Lexer/Cursor.swift | 251 +++--- .../SwiftParser/Lexer/LexemeSequence.swift | 2 +- .../SwiftParser/Lexer/RegexLiteralLexer.swift | 786 +++++++++++++++++ Sources/SwiftParser/TokenPrecedence.swift | 9 +- Sources/SwiftParser/TokenSpecSet.swift | 13 +- .../LexerDiagnosticMessages.swift | 6 + .../MissingNodesError.swift | 2 + .../MissingTokenError.swift | 20 +- .../ParserDiagnosticMessages.swift | 8 +- Sources/SwiftSyntax/TokenDiagnostic.swift | 6 + .../ClassifiedSyntaxTreePrinter.swift | 1 + Tests/SwiftParserTest/DeclarationTests.swift | 5 +- Tests/SwiftParserTest/ExpressionTests.swift | 8 - Tests/SwiftParserTest/LexerTests.swift | 223 ++++- Tests/SwiftParserTest/RegexLiteralTests.swift | 826 ++++++++++++++++++ ...orwardSlashRegexSkippingInvalidTests.swift | 124 +-- .../ForwardSlashRegexSkippingTests.swift | 100 +-- .../translated/ForwardSlashRegexTests.swift | 219 ++--- .../RegexParseEndOfBufferTests.swift | 7 +- .../translated/RegexParseErrorTests.swift | 38 +- 26 files changed, 2160 insertions(+), 577 deletions(-) create mode 100644 Sources/SwiftParser/Lexer/RegexLiteralLexer.swift create mode 100644 Tests/SwiftParserTest/RegexLiteralTests.swift diff --git a/CodeGeneration/Sources/SyntaxSupport/Classification.swift b/CodeGeneration/Sources/SyntaxSupport/Classification.swift index 860218fb2fe..0b1f5337734 100644 --- a/CodeGeneration/Sources/SyntaxSupport/Classification.swift +++ b/CodeGeneration/Sources/SyntaxSupport/Classification.swift @@ -58,6 +58,7 @@ public let SYNTAX_CLASSIFICATIONS: [SyntaxClassification] = [ SyntaxClassification(name: "ObjectLiteral", description: "An image, color, etc. literal."), SyntaxClassification(name: "OperatorIdentifier", description: "An identifier referring to an operator."), SyntaxClassification(name: "PoundDirectiveKeyword", description: "A `#` keyword like `#warning`."), + SyntaxClassification(name: "RegexLiteral", description: "A regex literal, including multiline regex literals."), SyntaxClassification(name: "StringInterpolationAnchor", description: "The opening and closing parenthesis of string interpolation."), SyntaxClassification(name: "StringLiteral", description: "A string literal including multiline string literals."), SyntaxClassification(name: "TypeIdentifier", description: "An identifier referring to a type."), diff --git a/CodeGeneration/Sources/SyntaxSupport/ExprNodes.swift b/CodeGeneration/Sources/SyntaxSupport/ExprNodes.swift index 9cb05403b41..d6bb093ccfe 100644 --- a/CodeGeneration/Sources/SyntaxSupport/ExprNodes.swift +++ b/CodeGeneration/Sources/SyntaxSupport/ExprNodes.swift @@ -1199,9 +1199,27 @@ public let EXPR_NODES: [Node] = [ kind: "Expr", children: [ Child( - name: "Regex", - kind: .token(choices: [.token(tokenKind: "RegexLiteralToken")]) - ) + name: "OpeningPounds", + kind: .token(choices: [.token(tokenKind: "ExtendedRegexDelimiterToken")]), + isOptional: true + ), + Child( + name: "OpenSlash", + kind: .token(choices: [.token(tokenKind: "RegexSlashToken")]) + ), + Child( + name: "RegexPattern", + kind: .token(choices: [.token(tokenKind: "RegexLiteralPatternToken")]) + ), + Child( + name: "CloseSlash", + kind: .token(choices: [.token(tokenKind: "RegexSlashToken")]) + ), + Child( + name: "ClosingPounds", + kind: .token(choices: [.token(tokenKind: "ExtendedRegexDelimiterToken")]), + isOptional: true + ), ] ), diff --git a/CodeGeneration/Sources/SyntaxSupport/TokenSpec.swift b/CodeGeneration/Sources/SyntaxSupport/TokenSpec.swift index c484a59fc65..2d9fc349d54 100644 --- a/CodeGeneration/Sources/SyntaxSupport/TokenSpec.swift +++ b/CodeGeneration/Sources/SyntaxSupport/TokenSpec.swift @@ -178,6 +178,7 @@ public let SYNTAX_TOKENS: [TokenSpec] = [ PunctuatorSpec(name: "Ellipsis", kind: "ellipsis", text: "..."), PunctuatorSpec(name: "Equal", kind: "equal", text: "=", requiresLeadingSpace: true, requiresTrailingSpace: true), PunctuatorSpec(name: "ExclamationMark", kind: "exclaim_postfix", text: "!"), + MiscSpec(name: "ExtendedRegexDelimiter", kind: "extended_regex_delimiter", nameForDiagnostics: "extended delimiter", classification: "RegexLiteral"), LiteralSpec(name: "FloatingLiteral", kind: "floating_literal", nameForDiagnostics: "floating literal", classification: "FloatingLiteral"), MiscSpec(name: "Identifier", kind: "identifier", nameForDiagnostics: "identifier", classification: "Identifier"), PunctuatorSpec(name: "InfixQuestionMark", kind: "question_infix", text: "?"), @@ -202,7 +203,8 @@ public let SYNTAX_TOKENS: [TokenSpec] = [ PunctuatorSpec(name: "PrefixAmpersand", kind: "amp_prefix", text: "&"), MiscSpec(name: "PrefixOperator", kind: "oper_prefix", nameForDiagnostics: "prefix operator", classification: "OperatorIdentifier"), MiscSpec(name: "RawStringDelimiter", kind: "raw_string_delimiter", nameForDiagnostics: "raw string delimiter"), - LiteralSpec(name: "RegexLiteral", kind: "regex_literal", nameForDiagnostics: "regex literal"), + MiscSpec(name: "RegexLiteralPattern", kind: "regex_literal_pattern", nameForDiagnostics: "regex pattern", classification: "RegexLiteral"), + PunctuatorSpec(name: "RegexSlash", kind: "regex_slash", text: "/", classification: "RegexLiteral"), PunctuatorSpec(name: "RightAngle", kind: "r_angle", text: ">"), PunctuatorSpec(name: "RightBrace", kind: "r_brace", text: "}"), PunctuatorSpec(name: "RightParen", kind: "r_paren", text: ")"), diff --git a/Sources/SwiftParser/CMakeLists.txt b/Sources/SwiftParser/CMakeLists.txt index 04cd4f94900..6aae28cb5a0 100644 --- a/Sources/SwiftParser/CMakeLists.txt +++ b/Sources/SwiftParser/CMakeLists.txt @@ -42,6 +42,7 @@ add_swift_host_library(SwiftParser Lexer/Lexeme.swift Lexer/LexemeSequence.swift Lexer/Lexer.swift + Lexer/RegexLiteralLexer.swift Lexer/UnicodeScalarExtensions.swift ) diff --git a/Sources/SwiftParser/Declarations.swift b/Sources/SwiftParser/Declarations.swift index 36319c37477..4e1e8c5d8b4 100644 --- a/Sources/SwiftParser/Declarations.swift +++ b/Sources/SwiftParser/Declarations.swift @@ -1315,28 +1315,6 @@ extension Parser { } extension Parser { - /// Are we at a regular expression literal that could act as an operator? - private mutating func atRegexLiteralThatCouldBeAnOperator() -> Bool { - guard self.at(.regexLiteral) else { - return false - } - - /// Try to re-lex at regex literal as an operator. If it succeeds and - /// consumes the entire regex literal, we're done. - return self.currentToken.tokenText.withBuffer { - (buffer: UnsafeBufferPointer) -> Bool in - var cursor = Lexer.Cursor(input: buffer, previous: 0) - guard buffer[0] == UInt8(ascii: "/") else { return false } - switch cursor.lexOperatorIdentifier(sourceBufferStart: cursor).tokenKind { - case .unknown: - return false - - default: - return cursor.input.isEmpty - } - } - } - @_spi(RawSyntax) public mutating func parseFuncDeclaration( _ attrs: DeclAttributes, @@ -1345,7 +1323,7 @@ extension Parser { let (unexpectedBeforeFuncKeyword, funcKeyword) = self.eat(handle) let unexpectedBeforeIdentifier: RawUnexpectedNodesSyntax? let identifier: RawTokenSyntax - if self.at(anyIn: Operator.self) != nil || self.at(.exclamationMark, .prefixAmpersand) || self.atRegexLiteralThatCouldBeAnOperator() { + if self.at(anyIn: Operator.self) != nil || self.at(.exclamationMark, .prefixAmpersand) { var name = self.currentToken.tokenText if name.count > 1 && name.hasSuffix("<") && self.peek().rawTokenKind == .identifier { name = SyntaxText(rebasing: name.dropLast()) diff --git a/Sources/SwiftParser/Expressions.swift b/Sources/SwiftParser/Expressions.swift index 77afcb5f691..a5cade88afa 100644 --- a/Sources/SwiftParser/Expressions.swift +++ b/Sources/SwiftParser/Expressions.swift @@ -1170,7 +1170,7 @@ extension Parser { ) case (.rawStringDelimiter, _)?, (.stringQuote, _)?, (.multilineStringQuote, _)?, (.singleQuote, _)?: return RawExprSyntax(self.parseStringLiteral()) - case (.regexLiteral, _)?: + case (.extendedRegexDelimiter, _)?, (.regexSlash, _)?: return RawExprSyntax(self.parseRegexLiteral()) case (.nilKeyword, let handle)?: let nilKeyword = self.eat(handle) @@ -1433,13 +1433,32 @@ extension Parser { /// Grammar /// ======= /// - /// regular-expression-literal → '\' `Any valid regular expression characters` '\' + /// regular-expression-literal → '#'* '/' `Any valid regular expression characters` '/' '#'* @_spi(RawSyntax) public mutating func parseRegexLiteral() -> RawRegexLiteralExprSyntax { - let (unexpectedBeforeLiteral, literal) = self.expect(.regexLiteral) + // See if we have an opening set of pounds. + let openPounds = self.consume(if: .extendedRegexDelimiter) + + // Parse the opening slash. + let (unexpectedBeforeSlash, openSlash) = self.expect(.regexSlash) + + // Parse the pattern and closing slash, avoiding recovery or leading trivia + // as the lexer should provide the tokens exactly in order without trivia, + // otherwise they should be treated as missing. + let pattern = self.expectWithoutRecoveryOrLeadingTrivia(.regexLiteralPattern) + let closeSlash = self.expectWithoutRecoveryOrLeadingTrivia(.regexSlash) + + // Finally, parse a closing set of pounds. + let (unexpectedBeforeClosePounds, closePounds) = parsePoundDelimiter(.extendedRegexDelimiter, matching: openPounds) + return RawRegexLiteralExprSyntax( - unexpectedBeforeLiteral, - regex: literal, + openingPounds: openPounds, + unexpectedBeforeSlash, + openSlash: openSlash, + regexPattern: pattern, + closeSlash: closeSlash, + unexpectedBeforeClosePounds, + closingPounds: closePounds, arena: self.arena ) } diff --git a/Sources/SwiftParser/Lexer/Cursor.swift b/Sources/SwiftParser/Lexer/Cursor.swift index 2cbbc26a176..4d618b3edd9 100644 --- a/Sources/SwiftParser/Lexer/Cursor.swift +++ b/Sources/SwiftParser/Lexer/Cursor.swift @@ -49,10 +49,20 @@ extension Lexer.Cursor { /// New entries in the state stack are added when: /// - A string literal is entered /// - A string interpolation inside is entered + /// - A regex literal is being lexed + /// - A narrow case for 'try?' and 'try!' to ensure correct regex lexing enum State { /// Normal top-level lexing mode case normal + /// A narrow mode that's used for 'try?' and 'try!' to ensure we prefer to + /// lex a regex literal rather than a binary operator. This is needed as the + /// last token will be a postfix operator, which would normally indicate a + /// binary operator is expected next, but in this case we know it must be an + /// expression. See the comment in `tryScanOperatorAsRegexLiteral` for more info. + /// NOTE: This is a complete hack, do not add new uses of this. + case preferRegexOverBinaryOperator + /// The lexer has lexed `delimiterLength` raw string delimiters '##' but not /// the string quote itself. case afterRawStringDelimiter(delimiterLength: Int) @@ -82,11 +92,15 @@ extension Lexer.Cursor { /// `stringInterpolationStart` points to the first character inside the interpolation. case inStringInterpolation(stringLiteralKind: StringLiteralKind, parenCount: Int) + /// We have encountered a regex literal, and have its tokens to work + /// through. + case inRegexLiteral(index: UInt8, lexemes: UnsafePointer) + /// The mode in which leading trivia should be lexed for this state or `nil` /// if no trivia should be lexed. func leadingTriviaLexingMode(cursor: Lexer.Cursor) -> TriviaLexingMode? { switch self { - case .normal: return .normal + case .normal, .preferRegexOverBinaryOperator: return .normal case .afterRawStringDelimiter: return nil case .inStringLiteral: return nil case .afterStringLiteral: return nil @@ -99,6 +113,7 @@ extension Lexer.Cursor { case .singleLine, .singleQuote: return .noNewlines case .multiLine: return .normal } + case .inRegexLiteral: return nil } } @@ -106,13 +121,14 @@ extension Lexer.Cursor { /// if no trivia should be lexed. func trailingTriviaLexingMode(cursor: Lexer.Cursor) -> TriviaLexingMode? { switch self { - case .normal: return .noNewlines + case .normal, .preferRegexOverBinaryOperator: return .noNewlines case .afterRawStringDelimiter: return nil case .inStringLiteral: return nil case .afterStringLiteral: return nil case .afterClosingStringQuote: return nil case .inStringInterpolationStart: return nil case .inStringInterpolation: return .noNewlines + case .inRegexLiteral: return nil } } @@ -122,13 +138,14 @@ extension Lexer.Cursor { /// hitting a newline. var shouldPopStateWhenReachingNewlineInTrailingTrivia: Bool { switch self { - case .normal: return false + case .normal, .preferRegexOverBinaryOperator: return false case .afterRawStringDelimiter: return false case .inStringLiteral(kind: let stringLiteralKind, delimiterLength: _): return stringLiteralKind != .multiLine case .afterStringLiteral: return false case .afterClosingStringQuote: return false case .inStringInterpolationStart: return false case .inStringInterpolation: return false + case .inRegexLiteral: return false } } } @@ -166,6 +183,8 @@ extension Lexer.Cursor { } } topState = newState + case .pushRegexLexemes(let index, let lexemes): + perform(stateTransition: .push(newState: .inRegexLiteral(index: index, lexemes: lexemes.allocate(in: stateAllocator))), stateAllocator: stateAllocator) case .replace(newState: let newState): topState = newState case .pop: @@ -189,9 +208,12 @@ extension Lexer.Cursor { /// The position in the token at which the diagnostic is. let position: Lexer.Cursor.Position - init(_ kind: TokenDiagnostic.Kind, position: Lexer.Cursor) { + init(_ kind: TokenDiagnostic.Kind, position: Lexer.Cursor.Position) { self.kind = kind - self.position = position.position + self.position = position + } + init(_ kind: TokenDiagnostic.Kind, position: Lexer.Cursor) { + self.init(kind, position: position.position) } func tokenDiagnostic(tokenStart: Lexer.Cursor) -> TokenDiagnostic { @@ -265,6 +287,11 @@ extension Lexer { enum StateTransition { /// Push a new state onto the state stack case push(newState: Cursor.State) + + /// Push a set of regex literal lexemes onto the state stack. This avoids + /// needing to plumb the state allocator through the lexer. + case pushRegexLexemes(index: UInt8, lexemes: RegexLiteralLexemes) + /// Replace the current state on the state stack by `newState` case replace(newState: Cursor.State) /// Pop a single state from the state stack. @@ -340,6 +367,10 @@ extension Lexer.Cursor { switch currentState { case .normal: result = lexNormal(sourceBufferStart: sourceBufferStart) + case .preferRegexOverBinaryOperator: + // In this state we lex a single token with the flag set, and then pop the state. + result = lexNormal(sourceBufferStart: sourceBufferStart, preferRegexOverBinaryOperator: true) + self.stateStack.perform(stateTransition: .pop, stateAllocator: stateAllocator) case .afterRawStringDelimiter(delimiterLength: let delimiterLength): result = lexAfterRawStringDelimiter(delimiterLength: delimiterLength) case .inStringLiteral(kind: let stringLiteralKind, delimiterLength: let delimiterLength): @@ -352,6 +383,8 @@ extension Lexer.Cursor { result = lexInStringInterpolationStart(stringLiteralKind: stringLiteralKind) case .inStringInterpolation(stringLiteralKind: let stringLiteralKind, parenCount: let parenCount): result = lexInStringInterpolation(stringLiteralKind: stringLiteralKind, parenCount: parenCount, sourceBufferStart: sourceBufferStart) + case .inRegexLiteral(let index, let lexemes): + result = lexInRegexLiteral(lexemes.pointee[index...], existingPtr: lexemes) } if let stateTransition = result.stateTransition { @@ -521,6 +554,14 @@ extension Lexer.Cursor.Position { self.input = UnsafeBufferPointer(rebasing: input) return c } + /// Advance the cursor position by `n` bytes. The offset must be valid. + func advanced(by n: Int) -> Self { + precondition(n > 0) + precondition(n <= self.input.count) + var input = self.input.dropFirst(n - 1) + let c = input.removeFirst() + return .init(input: UnsafeBufferPointer(rebasing: input), previous: c) + } } extension Lexer.Cursor { @@ -781,7 +822,10 @@ extension Lexer.Cursor { // MARK: - Main entry point extension Lexer.Cursor { - private mutating func lexNormal(sourceBufferStart: Lexer.Cursor) -> Lexer.Result { + private mutating func lexNormal( + sourceBufferStart: Lexer.Cursor, + preferRegexOverBinaryOperator: Bool = false + ) -> Lexer.Result { switch self.peek() { case UInt8(ascii: "@"): _ = self.advance(); return Lexer.Result(.atSign) case UInt8(ascii: "{"): _ = self.advance(); return Lexer.Result(.leftBrace) @@ -803,24 +847,20 @@ extension Lexer.Cursor { } // Try lex a regex literal. - if let token = self.tryLexRegexLiteral(sourceBufferStart: sourceBufferStart) { - return Lexer.Result(token) + if let result = self.lexRegexLiteral() { + return result } // Otherwise try lex a magic pound literal. return self.lexMagicPoundLiteral() - case UInt8(ascii: "/"): - // Try lex a regex literal. - if let token = self.tryLexRegexLiteral(sourceBufferStart: sourceBufferStart) { - return Lexer.Result(token) - } - // Otherwise try lex a magic pound literal. - return self.lexOperatorIdentifier(sourceBufferStart: sourceBufferStart) case UInt8(ascii: "!"), UInt8(ascii: "?"): if let result = lexPostfixOptionalChain(sourceBufferStart: sourceBufferStart) { return result } - return self.lexOperatorIdentifier(sourceBufferStart: sourceBufferStart) + return self.lexOperatorIdentifier( + sourceBufferStart: sourceBufferStart, + preferRegexOverBinaryOperator: preferRegexOverBinaryOperator + ) case UInt8(ascii: "<"): if self.is(offset: 1, at: "#"), @@ -828,7 +868,10 @@ extension Lexer.Cursor { { return result } - return self.lexOperatorIdentifier(sourceBufferStart: sourceBufferStart) + return self.lexOperatorIdentifier( + sourceBufferStart: sourceBufferStart, + preferRegexOverBinaryOperator: preferRegexOverBinaryOperator + ) case UInt8(ascii: "A"), UInt8(ascii: "B"), UInt8(ascii: "C"), UInt8(ascii: "D"), UInt8(ascii: "E"), UInt8(ascii: "F"), @@ -873,7 +916,10 @@ extension Lexer.Cursor { } if tmp.advance(if: { Unicode.Scalar($0).isOperatorStartCodePoint }) { - return self.lexOperatorIdentifier(sourceBufferStart: sourceBufferStart) + return self.lexOperatorIdentifier( + sourceBufferStart: sourceBufferStart, + preferRegexOverBinaryOperator: preferRegexOverBinaryOperator + ) } switch self.lexUnknown() { @@ -1672,9 +1718,9 @@ extension Lexer.Cursor { return .pop case .afterRawStringDelimiter(delimiterLength: let delimiterLength): return .replace(newState: .inStringLiteral(kind: kind, delimiterLength: delimiterLength)) - case .normal, .inStringInterpolation: + case .normal, .preferRegexOverBinaryOperator, .inStringInterpolation: return .push(newState: .inStringLiteral(kind: kind, delimiterLength: 0)) - default: + case .inRegexLiteral, .inStringLiteral, .afterClosingStringQuote, .inStringInterpolationStart: preconditionFailure("Unexpected currentState '\(currentState)' for 'stateTransitionAfterLexingStringQuote'") } } @@ -1916,6 +1962,14 @@ extension Lexer.Cursor { // Must be left bound, otherwise this isn't postfix. guard self.isLeftBound(sourceBufferStart: sourceBufferStart) else { return nil } + var transition: Lexer.StateTransition? + if previousKeyword == .try { + // If we have 'try' as the previous keyword kind, we have `try?` or `try!` + // and need to transition into the state where we prefer lexing a regex + // literal over a binary operator. See the comment in + // `tryScanOperatorAsRegexLiteral` for more info. + transition = .push(newState: .preferRegexOverBinaryOperator) + } let kind: RawTokenKind = { switch self.peek() { case UInt8(ascii: "!"): @@ -1927,10 +1981,13 @@ extension Lexer.Cursor { } }() _ = self.advance() - return Lexer.Result(kind) + return Lexer.Result(kind, stateTransition: transition) } - mutating func lexOperatorIdentifier(sourceBufferStart: Lexer.Cursor) -> Lexer.Result { + mutating func lexOperatorIdentifier( + sourceBufferStart: Lexer.Cursor, + preferRegexOverBinaryOperator: Bool + ) -> Lexer.Result { let tokStart = self let didStart = self.advance(if: { $0.isOperatorStartCodePoint }) precondition(didStart, "unexpected operator start") @@ -1945,15 +2002,31 @@ extension Lexer.Cursor { if text.hasPrefix("<#") && text.containsPlaceholderEnd() { break } - - // // If we are lexing a `/.../` regex literal, we don't consider `/` to be an - // // operator character. - // if ForwardSlashRegexMode != LexerForwardSlashRegexMode::None && - // CurPtr.peek() == UInt8(ascii: "/") { - // break - // } } while self.advance(if: { $0.isOperatorContinuationCodePoint }) + // Check to see if we have a regex literal starting in the operator. + do { + var ptr = tokStart + while ptr.input.baseAddress! < self.input.baseAddress! { + // Scan for the first '/' in the operator to see if it starts a regex + // literal. + guard ptr.is(at: "/") else { + _ = ptr.advance() + continue + } + guard + let result = self.tryLexOperatorAsRegexLiteral( + at: ptr, + operatorStart: tokStart, + operatorEnd: self, + sourceBufferStart: sourceBufferStart, + preferRegexOverBinaryOperator: preferRegexOverBinaryOperator + ) + else { break } + return result + } + } + if self.input.baseAddress! - tokStart.input.baseAddress! > 2 { // If there is a "//" or "/*" in the middle of an identifier token, // it starts a comment. @@ -2278,123 +2351,3 @@ extension Lexer.Cursor { return nil } } - -extension Lexer.Cursor { - mutating func tryLexRegexLiteral(sourceBufferStart: Lexer.Cursor) -> RawTokenKind? { - guard !self.isLeftBound(sourceBufferStart: sourceBufferStart) else { - return nil - } - - var tmp = self - var poundCount = 0 - var parenCount = 0 - - while tmp.advance(matching: "#") { - poundCount += 1 - } - - guard tmp.advance(matching: "/") else { - return nil - } - - // For `/.../` regex literals, we need to ban space and tab at the start of - // a regex to avoid ambiguity with operator chains, e.g: - // - // Builder { - // 0 - // / 1 / - // 2 - // } - // - if poundCount == 0 && tmp.is(at: " ", "\n", "\t") { - return nil - } - - var isMultiline = false - LOOP: while true { - switch tmp.peek() { - case UInt8(ascii: " "), UInt8(ascii: "\t"): - _ = tmp.advance() - case UInt8(ascii: "\n"), UInt8(ascii: "\r"): - isMultiline = true - break LOOP - default: - break LOOP - } - } - - var escaped = false - DELIMITLOOP: while true { - defer { escaped = false } - - let previousByte = tmp.previous - switch tmp.advance() { - case nil: - return nil - case UInt8(ascii: "/"): - // If we're at the end of the literal, peek ahead to see if the closing - // slash is actually the start of a comment. - if tmp.is(at: "/", "*") { - return nil - } - - var endLex = tmp - for _ in 0.. LexResult { + if cursor.isAtEndOfFile { + // We've hit the end of the buffer. In multi-line mode, we don't want to + // skip over what is likely otherwise valid Swift code, so resume from the + // first newline. + if let firstNewline = firstNewline { + cursor = firstNewline + } + return .unterminated + } + let charCursor = cursor + guard let char = cursor.advanceValidatingUTF8Character() else { + builder.recordPatternError(.invalidUtf8, at: cursor) + return .continue + } + switch char { + case "\n", "\r": + guard isMultiline else { + // Bump back the cursor to the newline to ensure it doesn't + // become part of the pattern token. + cursor = charCursor + return .unterminated + } + lastNewlineEnd = cursor + break + + case "\\" where !escaped: + // Advance again for an escape sequence. + return lexPatternCharacter(escaped: true) + + case "(" where !escaped && customCharacterClassDepth == 0: + groupDepth += 1 + + case ")" where !escaped && customCharacterClassDepth == 0: + guard groupDepth > 0 else { + // If we have an unbalanced ')', and this may not be a regex, don't + // lex as a regex. + if !mustBeRegex { + return .notARegex + } + break + } + groupDepth -= 1 + + case "[" where !escaped: + customCharacterClassDepth += 1 + + case "]" where !escaped: + if customCharacterClassDepth > 0 { + customCharacterClassDepth -= 1 + } + + case "\0": + builder.recordPatternError(.nulCharacter, at: charCursor) + break + + case let char + where char.isASCII && !char.isPrintableASCII && !(isMultiline && char == "\t"): + // Diagnose unprintable ASCII. + // Note that tabs are allowed in multi-line literals. + // TODO: This matches the string literal behavior, but should we allow + // tabs for single-line regex literals too? + builder.recordPatternError(.unprintableAsciiCharacter, at: charCursor) + break + + case " ", "\t": + if !escaped { + lastUnespacedSpaceOrTab = charCursor + } + break + + default: + break + } + return .continue + } + + /// Attempt to eat a the closing delimiter. + mutating func tryEatEnding() -> LexResult? { + let openPoundCount = builder.numOpenPounds + let slashBegin = cursor + var newCursor = cursor + + guard newCursor.advance(matching: "/") else { return nil } + let slashEnd = newCursor + candidateCloseSlashEnd = slashEnd.position + + // Try to eat closing pounds. Note we don't do this if we don't have any + // opening pounds (for recovery), as the builder currently bases the maximum + // token count off the presence of opening pounds, and it's not clear if + // recovery in that case is useful anyway. + var closePoundCount = 0 + if openPoundCount > 0 { + while newCursor.advance(matching: "#") { + closePoundCount += 1 + } + } + + // Make sure we have sufficient closing pounds. Note we can consume extra + // for better recovery. + guard closePoundCount >= openPoundCount else { return nil } + + // If we have a multi-line literal, make sure the closing delimiter + // appears alone on a newline. + if let lastNewlineEnd = lastNewlineEnd { + var delimScan = lastNewlineEnd + while delimScan.pointer < slashBegin.pointer { + if !delimScan.advance(matching: " ", "\t") { + builder.recordPatternError(.multilineRegexClosingNotOnNewline, at: slashBegin) + break + } + } + } + + if closePoundCount == 0 { + if let end = newCursor.peek() { + // If we're lexing `/.../`, treat it as unterminated if we ended on the + // opening of a comment. We prefer to lex the comment as it's more likely + // than not that is what the user is expecting. + switch UnicodeScalar(end) { + case "*", "/": + return .unterminated + default: + break + } + } + + // We also ban unespaced space and tab at the end of a regex literal if + // this might not be a regex. + if let lastUnespacedSpaceOrTab = lastUnespacedSpaceOrTab, + lastUnespacedSpaceOrTab.position.advanced(by: 1).pointer == slashBegin.position.pointer + { + if mustBeRegex { + // TODO: We ought to have a fix-it that suggests #/.../#. We could + // suggest escaping, but that would be wrong if the user has written (?x). + // TODO: Should we suggest #/.../# for space-as-first character too? + builder.recordPatternError(.spaceAtEndOfRegexLiteral, at: lastUnespacedSpaceOrTab) + } else { + return .notARegex + } + } + } + + builder.recordCloseSlash(endingAt: slashEnd.position) + if closePoundCount > 0 { + builder.recordClosePounds(count: closePoundCount) + } + cursor = newCursor + return .done + } + + mutating func lexImpl() -> LexResult { + // We can consume any number of pound signs. + var poundCount = 0 + while cursor.advance(matching: "#") { + poundCount += 1 + } + if poundCount > 0 { + builder.recordOpenPounds(count: poundCount) + } + + // Try to lex the opening delimiter. + guard cursor.advance(matching: "/") else { + return .notARegex + } + builder.recordOpenSlash() + + if !builder.hasPounds { + if let next = cursor.peek() { + switch UnicodeScalar(next) { + case " ", "\t": + // For `/.../` regex literals, we need to ban space and tab at the start + // of a regex to avoid ambiguity with operator chains, e.g: + // + // Builder { + // 0 + // / 1 / + // 2 + // } + // + if mustBeRegex { + // TODO: We ought to have a fix-it that inserts a backslash to escape. + builder.recordPatternError(.spaceAtStartOfRegexLiteral, at: cursor) + } else { + return .notARegex + } + case "*", "/": + // Start of a comment, not a regex. + return .notARegex + default: + break + } + } + } + + // If the delimiter allows multi-line, try skipping over any whitespace to a + // newline character. If we can do that, we enter multi-line mode. + if builder.hasPounds { + var newlineScan = cursor + while let next = newlineScan.peek() { + switch UnicodeScalar(next) { + case " ", "\t": + _ = newlineScan.advance() + continue + case "\n", "\r": + firstNewline = newlineScan + cursor = newlineScan + default: + break + } + break + } + } + + while true { + if let result = tryEatEnding() { + return result + } + switch lexPatternCharacter() { + case .continue: + continue + case let result: + return result + } + } + } + + mutating func lex() -> RegexLiteralLexemes? { + switch lexImpl() { + case .continue: + fatalError("Not a valid result") + case .notARegex: + return nil + case .unterminated where !mustBeRegex: + // If this may not be a regex, bail. + return nil + case .done, .unterminated: + // In both cases we can just return the lexemes. We'll diagnose when + // parsing. + return builder.finish( + at: cursor.position, + candidateCloseSlashEnd: candidateCloseSlashEnd + ) + } + } +} + +struct RegexLiteralLexemes { + private let builder: Builder + + fileprivate init(from builder: Builder) { + self.builder = builder + } + + struct Element { + var kind: Kind + var end: Lexer.Cursor.Position + var error: Lexer.Cursor.LexingDiagnostic? + } +} + +extension RegexLiteralLexemes { + /// Allocate the lexemes on a given bump pointer allocator. + func allocate(in allocator: BumpPtrAllocator) -> UnsafePointer { + let ptr = allocator.allocate(Self.self, count: 1).baseAddress! + ptr.initialize(to: self) + return UnsafePointer(ptr) + } +} + +extension RegexLiteralLexemes.Element { + /// The regex literal token kind, the raw value of which indicates its index. + enum Kind: UInt8 { + case openingPounds + case openingSlash + case pattern + case closingSlash + case closingPounds + } + /// Retrieve the actual token kind. + var tokenKind: RawTokenKind { + switch kind { + case .openingPounds, .closingPounds: + return .extendedRegexDelimiter + case .openingSlash, .closingSlash: + return .regexSlash + case .pattern: + return .regexLiteralPattern + } + } +} + +extension RegexLiteralLexemes.Element.Kind { + /// Construct the regex literal token kind from a given index, taking pounds + /// into account. + fileprivate init(at index: UInt8, hasPounds: Bool) { + if hasPounds { + // If we have pounds, we have 5 tokens maximum. + precondition(index < 5) + self.init(rawValue: index)! + } else { + // Otherwise, we have 3 tokens maximum, and start at the slash. + precondition(index < 3) + self.init(rawValue: index + 1)! + } + } +} + +extension RegexLiteralLexemes { + /// A builder type for the regex literal lexer. + /// + /// NOTE: This is stored for the regex literal lexer state, so should be kept + /// as small as possible. + fileprivate struct Builder { + private(set) var numOpenPounds: Int = 0 + private(set) var patternByteLength: Int = 0 + private(set) var numClosePounds: Int = 0 + + // The start position is split into its component input buffer and + // previous char to allow for a more optimized layout. + private let _startInput: UnsafeBufferPointer + + // The pattern diagnostic is split for a more optimized layout. + private var _patternErrorOffset: Int? + private var _patternErrorKind: TokenDiagnostic.Kind? + + private let _startPrevious: UInt8 + + /// The number of lexemes. This is a UInt8 as there can only be a maximum + /// of 5 lexemes for a regex literal (open pounds, open slash, pattern, + /// closing slash, closing pounds). + private(set) var lexemeCount: UInt8 = 0 + + init(startingAt start: Lexer.Cursor.Position) { + self._startInput = start.input + self._startPrevious = start.previous + } + var start: Lexer.Cursor.Position { + .init(input: _startInput, previous: _startPrevious) + } + + private(set) var patternError: Lexer.Cursor.LexingDiagnostic? { + get { + guard let _patternErrorKind = _patternErrorKind else { return nil } + let pos = start.advanced(by: _patternErrorOffset!) + return .init(_patternErrorKind, position: pos) + } + set { + guard let newValue = newValue else { + _patternErrorKind = nil + _patternErrorOffset = nil + return + } + _patternErrorKind = newValue.kind + _patternErrorOffset = start.distance(to: newValue.position) + } + } + var hasPounds: Bool { numOpenPounds > 0 } + } +} + +extension RegexLiteralLexemes.Builder { + typealias Element = RegexLiteralLexemes.Element + + /// The lexeme kind for the last lexed token, or `nil` if no token has been + /// lexed yet. + var lastLexemeKind: Element.Kind? { + if lexemeCount == 0 { return nil } + return .init(at: lexemeCount - 1, hasPounds: hasPounds) + } + + /// The end byte offset for a given regex token kind. + func endByteOffset(for kind: Element.Kind) -> Int { + switch kind { + case .openingPounds: + return numOpenPounds + case .openingSlash: + return numOpenPounds + 1 + case .pattern: + return numOpenPounds + 1 + patternByteLength + case .closingSlash: + return numOpenPounds + 1 + patternByteLength + 1 + case .closingPounds: + return numOpenPounds + 1 + patternByteLength + 1 + numClosePounds + } + } + + /// Retrieve the end cursor position for a given regex token kind. + func endCursorPosition(for kind: Element.Kind) -> Lexer.Cursor.Position { + start.advanced(by: endByteOffset(for: kind)) + } + + mutating func recordOpenPounds(count: Int) { + precondition(lastLexemeKind == nil) + numOpenPounds = count + lexemeCount += 1 + } + + mutating func recordOpenSlash() { + precondition(lastLexemeKind == nil || lastLexemeKind == .openingPounds) + lexemeCount += 1 + } + + private mutating func recordRegexPattern(byteLength: Int) { + precondition(lastLexemeKind == .openingSlash) + patternByteLength = byteLength + lexemeCount += 1 + } + + mutating func recordCloseSlash(endingAt closeSlashEnd: Lexer.Cursor.Position) { + precondition(lastLexemeKind == .openingSlash) + + // We use the close slash to compute the pattern length. + let patternStart = endCursorPosition(for: .openingSlash) + recordRegexPattern(byteLength: patternStart.distance(to: closeSlashEnd) - 1) + lexemeCount += 1 + } + + mutating func recordClosePounds(count: Int) { + precondition(lastLexemeKind == .closingSlash) + numClosePounds = count + lexemeCount += 1 + } + + mutating func recordPatternError( + _ kind: TokenDiagnostic.Kind, + at cursor: Lexer.Cursor + ) { + precondition(lastLexemeKind == .openingSlash) + patternError = .init(kind, position: cursor) + } + + /// Finish regex literal lexing. + mutating func finish( + at end: Lexer.Cursor.Position, + candidateCloseSlashEnd: Lexer.Cursor.Position? + ) -> RegexLiteralLexemes { + // If we ended up in the middle of a pattern, we have an unterminated + // literal. Make sure to record the pattern, and do some recovery for + // better diagnostics. + if lastLexemeKind == .openingSlash { + let patternStart = endCursorPosition(for: .openingSlash) + let byteLength = patternStart.distance(to: end) + + // If have an extended literal, we can do some recovery for unterminated + // cases by seeing if we have something that looks like the ending + // delimiter at the end of the pattern, and treating it as such. The + // parser will diagnose if it doesn't end up matching. + func inferClosingDelimiter() -> (slashEnd: Lexer.Cursor.Position, numPounds: Int)? { + guard hasPounds && byteLength > 0 else { return nil } + + // If the last characeter is '/', we can use that. + if UnicodeScalar(end.previous) == "/" { + return (end, numPounds: 0) + } + // If the last character is '#', scan from the candidate last slash to + // see if we only have '#' characters until the end. In such a case, + // we can claim those characters as part of the delimiter. + if UnicodeScalar(end.previous) == "#", + let candidateCloseSlashEnd = candidateCloseSlashEnd + { + var poundScan = candidateCloseSlashEnd + var numClosingPounds = 0 + while poundScan.pointer < end.pointer, + UnicodeScalar(poundScan.advance()!) == "#" + { + numClosingPounds += 1 + } + precondition(numClosingPounds < numOpenPounds, "Should have lexed this?") + + // Should be at the end now, otherwise this is something bogus in + // the middle of the pattern. + if poundScan.pointer == end.pointer { + return (candidateCloseSlashEnd, numClosingPounds) + } + } + return nil + } + if let (closeSlashEnd, numClosingPounds) = inferClosingDelimiter() { + recordCloseSlash(endingAt: closeSlashEnd) + if numClosingPounds > 0 { + recordClosePounds(count: numClosingPounds) + } + } else { + recordRegexPattern(byteLength: byteLength) + } + } + return .init(from: self) + } +} + +extension RegexLiteralLexemes: RandomAccessCollection { + typealias Index = UInt8 + + var startIndex: UInt8 { 0 } + var endIndex: UInt8 { builder.lexemeCount } + + /// Retrieve the token at the given index. + subscript(index: UInt8) -> Element { + let kind = Element.Kind(at: index, hasPounds: builder.hasPounds) + return .init( + kind: kind, + end: builder.endCursorPosition(for: kind), + error: kind == .pattern ? builder.patternError : nil + ) + } +} + +extension Lexer.Cursor { + /// A heuristic that determines whether the cursor is currently in a regex + /// literal position by looking at the previous token to determine if we're + /// expecting an expression, or a binary operator. + fileprivate func isInRegexLiteralPosition() -> Bool { + switch previousTokenKind { + // Can lex a regex literal at the start of the buffer. + case nil: + return true + + // Cannot lex at the end of the buffer. + case .eof: + return false + + // Prefix grammar that appears before an expression. + case .leftAngle, .leftBrace, .leftParen, .leftSquareBracket, .prefixOperator, .prefixAmpersand: + return true + + // Binary operators sequence expressions. + case .binaryOperator, .equal: + return true + + // Infix punctuation that generally separates expressions. + case .semicolon, .comma, .colon, .infixQuestionMark: + return true + + // Postfix grammar would expect an binary operator next. + case .postfixOperator, .exclamationMark, .postfixQuestionMark, .rightAngle, .rightBrace, .rightParen, .rightSquareBracket: + return false + + // Punctuation that does not sequence expressions. + case .arrow, .ellipsis, .period, .atSign, .pound, .backtick, .backslash: + return false + + case .keyword: + // There are a handful of keywords that are expressions, handle them. + // Otherwise, a regex literal can generally be parsed after a keyword. + switch previousKeyword! { + case .true, .false, .Any, .nil, .`self`, .`Self`, .super: + return false + default: + return true + } + + // Identifiers do not sequence expressions. + case .identifier, .dollarIdentifier, .wildcard: + return false + + // Literals are themselves expressions and therefore don't sequence expressions. + case .floatingLiteral, .integerLiteral: + return false + + // Pound keywords that do not generally sequence expressions. + case .poundAvailableKeyword, .poundSourceLocationKeyword, .poundUnavailableKeyword: + return false + + // Pound keywords that generally do sequence expressions. + case .poundIfKeyword, .poundElseKeyword, .poundElseifKeyword, .poundEndifKeyword: + return true + + // Bits of string/regex grammar, we can't start lexing a regex literal here. + case .extendedRegexDelimiter, .regexSlash, .regexLiteralPattern, .rawStringDelimiter, .stringQuote, .stringSegment, .multilineStringQuote, .singleQuote: + return false + + // Allow unknown for better recovery. + case .unknown: + return true + } + } +} + +extension Lexer.Cursor { + /// Scan for a regex literal, without advancing the cursor. Returns the regex + /// literal tokens scanned, or `nil` if there is no regex literal. + fileprivate func scanRegexLiteral(mustBeRegex: Bool) -> RegexLiteralLexemes? { + var lexer = RegexLiteralLexer(self, mustBeRegex: mustBeRegex) + return lexer.lex() + } + + /// Attempt to scan for a regex literal starting from within an operator we've + /// lexed. + fileprivate func tryScanOperatorAsRegexLiteral( + operatorStart: Lexer.Cursor, + operatorEnd: Lexer.Cursor, + sourceBufferStart: Lexer.Cursor, + preferRegexOverBinaryOperator: Bool + ) -> RegexLiteralLexemes? { + precondition(self.pointer >= operatorStart.pointer, "lexing before the operator?") + + let isLeftBound = operatorStart.isLeftBound(sourceBufferStart: sourceBufferStart) + let isRightBound = operatorEnd.isRightBound(isLeftBound: isLeftBound) + + // Must not be left bound, we should lex a postfix '/' instead. + guard !isLeftBound else { return nil } + + // If the previous token was 'func' or 'operator', the next token must + // be an operator, not a regex. This is needed to correctly handle cases + // like: + // + // operator /^/ + // postfix func /(lhs: Int) { 5/ } + // + // Re-lexing isn't a viable strategy as there could be unbalanced curly + // braces in the regex, which interferes with the lexical structure (e.g + // anything relying on the lexed tokens to correctly balance curly braces). + switch self.previousKeyword { + case .func, .operator: + return nil + default: + break + } + + // Handle cases where the '/' is part of what looks like a binary operator. + var mustBeRegex = false + if isLeftBound == isRightBound { + // The `preferRegexOverBinaryOperator` flag is set if we previously had a + // 'try?' or 'try!'. In that case, the previous token is a postfix + // operator, which would usually indicate that we're not in regex literal + // position (as we would typically expect a binary operator to follow a + // postfix operator, not an expression). However 'try' is special because + // it cannot appear on the LHS of a binary operator, so we know we must + // have a regex. + // + // This is needed to handle cases like `try? /^ x/`, which should be lexed + // as a regex. This can occur for cases like `try? /^ x/.wholeMatch(...)`. + if preferRegexOverBinaryOperator { + mustBeRegex = true + } + // If we are not in a regex literal position, and are not certain, then + // prefer lexing as a binary operator instead. + if !mustBeRegex && !operatorStart.isInRegexLiteralPosition() { + return nil + } + // For better recovery, we can confidently lex a regex literal if the + // previous token was a binary operator and we're in a binary operator + // position, as that would otherwise be illegal. + // TODO: We could probably expand this to other token kinds (or rely on + // `isInRegexLiteralPosition`) for better recovery, but we'd need to be + // wary not to interfere with unapplied operator parsing, which are binary + // operators in expression position. For now, this handles the most common + // case. + if self.previousTokenKind?.is(.binaryOperator, .equal) == true { + mustBeRegex = true + } + } + return scanRegexLiteral(mustBeRegex: mustBeRegex) + } +} + +extension Lexer.Cursor { + /// Attempt to lex a regex literal. Note this lexes confidently, without + /// applying various heuristics to avoid lexing a regex literal in ambiguous + /// cases. + mutating func lexRegexLiteral() -> Lexer.Result? { + guard let lexemes = scanRegexLiteral(mustBeRegex: true) else { return nil } + return lexInRegexLiteral(lexemes[...], existingPtr: nil) + } + + /// Attempt to lex an operator as a regex literal, returning the result, + /// or `nil` if a regex literal could not be lexed. + mutating func tryLexOperatorAsRegexLiteral( + at regexStart: Lexer.Cursor, + operatorStart: Lexer.Cursor, + operatorEnd: Lexer.Cursor, + sourceBufferStart: Lexer.Cursor, + preferRegexOverBinaryOperator: Bool + ) -> Lexer.Result? { + guard + let lexemes = regexStart.tryScanOperatorAsRegexLiteral( + operatorStart: operatorStart, + operatorEnd: operatorEnd, + sourceBufferStart: sourceBufferStart, + preferRegexOverBinaryOperator: preferRegexOverBinaryOperator + ) + else { return nil } + + if regexStart.pointer > operatorStart.pointer { + // If we started lexing in the middle of an operator, split off the prefix + // operator, and move the cursor to where the regex literal starts. + self.position = regexStart.position + return Lexer.Result( + .prefixOperator, + stateTransition: .pushRegexLexemes(index: 0, lexemes: lexemes) + ) + } else { + // Otherwise we just have a regex literal. We can call into + // `lexInRegexLiteral` to pop the first token and push the state. + return lexInRegexLiteral(lexemes[...], existingPtr: nil) + } + } + + /// Lex an already-lexed regex literal. If `existingPtr` is non-nil, this is + /// for an existing regex literal state on the lexer state stack. + mutating func lexInRegexLiteral( + _ lexemes: Slice, + existingPtr: UnsafePointer? + ) -> Lexer.Result { + // Given we have already lexed the regex literal, this is as simple as + // popping off the next token and moving the lexer up to its end position. + var lexemes = lexemes + let lexeme = lexemes.removeFirst() + + self.position = lexeme.end + + // The new index is now given by the slice start index (as we've removed + // the first element). + let index = lexemes.startIndex + + // Compute the new transition. + let transition: Lexer.StateTransition? + if let existingPtr = existingPtr { + transition = lexemes.isEmpty ? .pop : .replace(newState: .inRegexLiteral(index: index, lexemes: existingPtr)) + } else { + transition = lexemes.isEmpty ? nil : .pushRegexLexemes(index: index, lexemes: lexemes.base) + } + return .init(lexeme.tokenKind, error: lexeme.error, stateTransition: transition) + } +} diff --git a/Sources/SwiftParser/TokenPrecedence.swift b/Sources/SwiftParser/TokenPrecedence.swift index 55fc529cc70..93049d329a3 100644 --- a/Sources/SwiftParser/TokenPrecedence.swift +++ b/Sources/SwiftParser/TokenPrecedence.swift @@ -115,15 +115,15 @@ public enum TokenPrecedence: Comparable { self = .unknownToken // MARK: Identifier like case // Literals - .floatingLiteral, .integerLiteral, .regexLiteral, + .floatingLiteral, .integerLiteral, // Pound literals .poundAvailableKeyword, .poundSourceLocationKeyword, .poundUnavailableKeyword, // Identifiers .dollarIdentifier, .identifier, // '_' can occur in types to replace a type identifier .wildcard, - // String segment, string interpolation anchor and pound don't really fit anywhere else - .pound, .stringSegment: + // String segment, string interpolation anchor, pound, and regex pattern don't really fit anywhere else + .pound, .stringSegment, .regexLiteralPattern: self = .identifierLike // MARK: Expr keyword @@ -139,7 +139,8 @@ public enum TokenPrecedence: Comparable { self = .weakBracketed(closingDelimiter: .rightSquareBracket) case .leftAngle: self = .weakBracketed(closingDelimiter: .rightAngle) - case .multilineStringQuote, .rawStringDelimiter, .singleQuote, .stringQuote: + case .multilineStringQuote, .rawStringDelimiter, .singleQuote, .stringQuote, + .regexSlash, .extendedRegexDelimiter: self = .weakBracketed(closingDelimiter: tokenKind) case // Chaining punctuators .infixQuestionMark, .period, .postfixQuestionMark, .exclamationMark, diff --git a/Sources/SwiftParser/TokenSpecSet.swift b/Sources/SwiftParser/TokenSpecSet.swift index 9da678e61fd..96abcc6d484 100644 --- a/Sources/SwiftParser/TokenSpecSet.swift +++ b/Sources/SwiftParser/TokenSpecSet.swift @@ -329,7 +329,6 @@ enum OperatorLike: TokenSpecSet { case postfixQuestionMark case equal case arrow - case regexLiteral // regex literals can look like operators, e.g. '/^/' init?(lexeme: Lexer.Lexeme) { if let op = Operator(lexeme: lexeme) { @@ -342,7 +341,6 @@ enum OperatorLike: TokenSpecSet { case .postfixQuestionMark: self = .postfixQuestionMark case .equal: self = .equal case .arrow: self = .arrow - case .regexLiteral: self = .regexLiteral default: return nil } } @@ -354,7 +352,6 @@ enum OperatorLike: TokenSpecSet { .postfixQuestionMark, .equal, .arrow, - .regexLiteral, ] } @@ -366,7 +363,6 @@ enum OperatorLike: TokenSpecSet { case .postfixQuestionMark: return TokenSpec(.postfixQuestionMark, remapping: .postfixOperator) case .equal: return TokenSpec(.equal, remapping: .binaryOperator) case .arrow: return TokenSpec(.arrow, remapping: .binaryOperator) - case .regexLiteral: return TokenSpec(.regexLiteral, remapping: .binaryOperator, recoveryPrecedence: TokenPrecedence(nonKeyword: .binaryOperator)) } } } @@ -588,7 +584,8 @@ enum PrimaryExpressionStart: TokenSpecSet { case pound case poundAvailableKeyword // For recovery case poundUnavailableKeyword // For recovery - case regexLiteral + case regexSlash + case extendedRegexDelimiter case selfKeyword case superKeyword case trueKeyword @@ -616,7 +613,8 @@ enum PrimaryExpressionStart: TokenSpecSet { case TokenSpec(.pound): self = .pound case TokenSpec(.poundAvailableKeyword): self = .poundAvailableKeyword case TokenSpec(.poundUnavailableKeyword): self = .poundUnavailableKeyword - case TokenSpec(.regexLiteral): self = .regexLiteral + case TokenSpec(.regexSlash): self = .regexSlash + case TokenSpec(.extendedRegexDelimiter): self = .extendedRegexDelimiter case TokenSpec(.self): self = .selfKeyword case TokenSpec(.super): self = .superKeyword case TokenSpec(.true): self = .trueKeyword @@ -647,7 +645,8 @@ enum PrimaryExpressionStart: TokenSpecSet { case .pound: return .pound case .poundAvailableKeyword: return .poundAvailableKeyword case .poundUnavailableKeyword: return .poundUnavailableKeyword - case .regexLiteral: return .regexLiteral + case .regexSlash: return .regexSlash + case .extendedRegexDelimiter: return .extendedRegexDelimiter case .selfKeyword: return .keyword(.self) case .superKeyword: return .keyword(.super) case .trueKeyword: return .keyword(.true) diff --git a/Sources/SwiftParserDiagnostics/LexerDiagnosticMessages.swift b/Sources/SwiftParserDiagnostics/LexerDiagnosticMessages.swift index 09307f8ed0e..65478cad737 100644 --- a/Sources/SwiftParserDiagnostics/LexerDiagnosticMessages.swift +++ b/Sources/SwiftParserDiagnostics/LexerDiagnosticMessages.swift @@ -54,6 +54,9 @@ public enum StaticTokenError: String, DiagnosticMessage { case sourceConflictMarker = "source control conflict marker in source file" case unexpectedBlockCommentEnd = "unexpected end of block comment" case unicodeCurlyQuote = #"unicode curly quote found; use '"' instead"# + case spaceAtStartOfRegexLiteral = "bare slash regex literal may not start with space" + case spaceAtEndOfRegexLiteral = "bare slash regex literal may not end with space" + case multilineRegexClosingNotOnNewline = "multi-line regex closing delimiter must appear on new line" case unprintableAsciiCharacter = "unprintable ASCII character found in source file" public var message: String { self.rawValue } @@ -163,6 +166,9 @@ public extension SwiftSyntax.TokenDiagnostic { case .sourceConflictMarker: return StaticTokenError.sourceConflictMarker case .unexpectedBlockCommentEnd: return StaticTokenError.unexpectedBlockCommentEnd case .unicodeCurlyQuote: return StaticTokenError.unicodeCurlyQuote + case .spaceAtStartOfRegexLiteral: return StaticTokenError.spaceAtStartOfRegexLiteral + case .spaceAtEndOfRegexLiteral: return StaticTokenError.spaceAtEndOfRegexLiteral + case .multilineRegexClosingNotOnNewline: return StaticTokenError.multilineRegexClosingNotOnNewline case .unprintableAsciiCharacter: return StaticTokenError.unprintableAsciiCharacter } } diff --git a/Sources/SwiftParserDiagnostics/MissingNodesError.swift b/Sources/SwiftParserDiagnostics/MissingNodesError.swift index 2c9cde23505..705a6e67414 100644 --- a/Sources/SwiftParserDiagnostics/MissingNodesError.swift +++ b/Sources/SwiftParserDiagnostics/MissingNodesError.swift @@ -166,6 +166,8 @@ fileprivate extension TokenKind { return .leftSquareBracket case .stringQuote, .multilineStringQuote, .rawStringDelimiter: return self + case .regexSlash, .extendedRegexDelimiter: + return self default: return nil } diff --git a/Sources/SwiftParserDiagnostics/MissingTokenError.swift b/Sources/SwiftParserDiagnostics/MissingTokenError.swift index fadcc2352af..c18699209a7 100644 --- a/Sources/SwiftParserDiagnostics/MissingTokenError.swift +++ b/Sources/SwiftParserDiagnostics/MissingTokenError.swift @@ -34,8 +34,8 @@ extension ParseDiagnosticsGenerator { handled = handleInvalidMultilineStringQuote(invalidToken: invalidToken, missingToken: missingToken, invalidTokenContainer: invalidTokenContainer) case (.period, .period): handled = handleInvalidPeriod(invalidToken: invalidToken, missingToken: missingToken, invalidTokenContainer: invalidTokenContainer) - case (.rawStringDelimiter, .rawStringDelimiter): - handled = handleInvalidRawStringDelimiter(invalidToken: invalidToken, missingToken: missingToken, invalidTokenContainer: invalidTokenContainer) + case (.rawStringDelimiter, .rawStringDelimiter), (.extendedRegexDelimiter, .extendedRegexDelimiter): + handled = handleInvalidPoundDelimiter(invalidToken: invalidToken, missingToken: missingToken, invalidTokenContainer: invalidTokenContainer) default: handled = false } @@ -122,19 +122,25 @@ extension ParseDiagnosticsGenerator { return true } - private func handleInvalidRawStringDelimiter(invalidToken: TokenSyntax, missingToken: TokenSyntax, invalidTokenContainer: UnexpectedNodesSyntax) -> Bool { + private func handleInvalidPoundDelimiter( + invalidToken: TokenSyntax, + missingToken: TokenSyntax, + invalidTokenContainer: UnexpectedNodesSyntax + ) -> Bool { + let isTooMany = invalidToken.contentLength > missingToken.contentLength let message: DiagnosticMessage if missingToken.parent?.is(ExpressionSegmentSyntax.self) == true { message = .tooManyRawStringDelimitersToStartInterpolation } else { + let parent = missingToken.parent! precondition( - missingToken.parent?.is(StringLiteralExprSyntax.self) == true, - "Raw string delimiters should only occur in string interpolation and at the end of a string literal" + parent.is(StringLiteralExprSyntax.self) || parent.is(RegexLiteralExprSyntax.self), + "Raw string delimiters should only occur in string interpolation and at the end of a string or regex literal" ) - message = .tooManyClosingRawStringDelimiters + message = isTooMany ? StaticParserError.tooManyClosingPoundDelimiters : .tooFewClosingPoundDelimiters } let fixIt = FixIt( - message: .removeExtraneousDelimiters, + message: isTooMany ? .removeExtraneousDelimiters : .insertExtraClosingPounds, changes: [ .makeMissing(invalidToken), .makePresentBeforeTrivia(missingToken), diff --git a/Sources/SwiftParserDiagnostics/ParserDiagnosticMessages.swift b/Sources/SwiftParserDiagnostics/ParserDiagnosticMessages.swift index db70ffecdac..3cd88f30a2a 100644 --- a/Sources/SwiftParserDiagnostics/ParserDiagnosticMessages.swift +++ b/Sources/SwiftParserDiagnostics/ParserDiagnosticMessages.swift @@ -197,9 +197,12 @@ extension DiagnosticMessage where Self == StaticParserError { public static var subscriptsCannotHaveNames: Self { .init("subscripts cannot have a name") } - public static var tooManyClosingRawStringDelimiters: Self { + public static var tooManyClosingPoundDelimiters: Self { .init("too many '#' characters in closing delimiter") } + public static var tooFewClosingPoundDelimiters: Self { + .init("expected additional '#' characters in closing delimiter") + } public static var tooManyRawStringDelimitersToStartInterpolation: Self { .init("too many '#' characters to start string interpolation") } @@ -519,6 +522,9 @@ extension FixItMessage where Self == StaticParserFixIt { public static var removeExtraneousDelimiters: Self { .init("remove extraneous delimiters") } + public static var insertExtraClosingPounds: Self { + .init("insert additional closing '#' delimiters") + } public static var removeExtraneousWhitespace: Self { .init("remove whitespace") } diff --git a/Sources/SwiftSyntax/TokenDiagnostic.swift b/Sources/SwiftSyntax/TokenDiagnostic.swift index 5b2be370e5a..64ab846ebff 100644 --- a/Sources/SwiftSyntax/TokenDiagnostic.swift +++ b/Sources/SwiftSyntax/TokenDiagnostic.swift @@ -49,6 +49,9 @@ public struct TokenDiagnostic: Hashable { case unexpectedBlockCommentEnd case unicodeCurlyQuote case unprintableAsciiCharacter + case spaceAtStartOfRegexLiteral + case spaceAtEndOfRegexLiteral + case multilineRegexClosingNotOnNewline } public let kind: Kind @@ -122,6 +125,9 @@ public struct TokenDiagnostic: Hashable { case .unexpectedBlockCommentEnd: return .error case .unicodeCurlyQuote: return .error case .unprintableAsciiCharacter: return .error + case .spaceAtStartOfRegexLiteral: return .error + case .spaceAtEndOfRegexLiteral: return .error + case .multilineRegexClosingNotOnNewline: return .error } } } diff --git a/Sources/lit-test-helper/ClassifiedSyntaxTreePrinter.swift b/Sources/lit-test-helper/ClassifiedSyntaxTreePrinter.swift index 8a1b0bb2a73..c6739d09b78 100644 --- a/Sources/lit-test-helper/ClassifiedSyntaxTreePrinter.swift +++ b/Sources/lit-test-helper/ClassifiedSyntaxTreePrinter.swift @@ -26,6 +26,7 @@ extension SyntaxClassification { case .floatingLiteral: return "float" case .stringLiteral: return "str" case .stringInterpolationAnchor: return "anchor" + case .regexLiteral: return "regex" case .poundDirectiveKeyword: return "#kw" case .buildConfigId: return "#id" case .attribute: return "attr-builtin" diff --git a/Tests/SwiftParserTest/DeclarationTests.swift b/Tests/SwiftParserTest/DeclarationTests.swift index 66c544fc884..1bd21917857 100644 --- a/Tests/SwiftParserTest/DeclarationTests.swift +++ b/Tests/SwiftParserTest/DeclarationTests.swift @@ -66,10 +66,9 @@ final class DeclarationTests: XCTestCase { assertParse("func /^/ (lhs: Int, rhs: Int) -> Int { 1 / 2 }") assertParse( - "func 1️⃣/^notoperator^/ (lhs: Int, rhs: Int) -> Int { 1 / 2 }", + "func /^1️⃣notoperator^/ (lhs: Int, rhs: Int) -> Int { 1 / 2 }", diagnostics: [ - DiagnosticSpec(message: "expected identifier in function"), - DiagnosticSpec(message: "unexpected code '/^notoperator^/' before parameter clause"), + DiagnosticSpec(message: "unexpected code 'notoperator^/' before parameter clause") ] ) diff --git a/Tests/SwiftParserTest/ExpressionTests.swift b/Tests/SwiftParserTest/ExpressionTests.swift index 6b822a57649..2654209a715 100644 --- a/Tests/SwiftParserTest/ExpressionTests.swift +++ b/Tests/SwiftParserTest/ExpressionTests.swift @@ -304,14 +304,6 @@ final class ExpressionTests: XCTestCase { ) } - func testRegexLiteral() { - assertParse( - #""" - /(?[[:alpha:]]\w*) = (?[0-9A-F]+)/ - """# - ) - } - func testInitializerExpression() { assertParse("Lexer.Cursor(input: input, previous: 0)") } diff --git a/Tests/SwiftParserTest/LexerTests.swift b/Tests/SwiftParserTest/LexerTests.swift index c386b840344..047b7f9fb9e 100644 --- a/Tests/SwiftParserTest/LexerTests.swift +++ b/Tests/SwiftParserTest/LexerTests.swift @@ -337,19 +337,29 @@ public class LexerTests: XCTestCase { assertLexemes( "/abc/", lexemes: [ - LexemeSpec(.regexLiteral, text: "/abc/") + LexemeSpec(.regexSlash, text: "/"), + LexemeSpec(.regexLiteralPattern, text: "abc"), + LexemeSpec(.regexSlash, text: "/"), ] ) assertLexemes( "#/abc/#", lexemes: [ - LexemeSpec(.regexLiteral, text: "#/abc/#") + LexemeSpec(.extendedRegexDelimiter, text: "#"), + LexemeSpec(.regexSlash, text: "/"), + LexemeSpec(.regexLiteralPattern, text: "abc"), + LexemeSpec(.regexSlash, text: "/"), + LexemeSpec(.extendedRegexDelimiter, text: "#"), ] ) assertLexemes( "###/abc/###", lexemes: [ - LexemeSpec(.regexLiteral, text: "###/abc/###") + LexemeSpec(.extendedRegexDelimiter, text: "###"), + LexemeSpec(.regexSlash, text: "/"), + LexemeSpec(.regexLiteralPattern, text: "abc"), + LexemeSpec(.regexSlash, text: "/"), + LexemeSpec(.extendedRegexDelimiter, text: "###"), ] ) assertLexemes( @@ -360,45 +370,69 @@ public class LexerTests: XCTestCase { /# """, lexemes: [ - LexemeSpec(.regexLiteral, text: "#/\na\nb\n/#") + LexemeSpec(.extendedRegexDelimiter, text: "#"), + LexemeSpec(.regexSlash, text: "/"), + LexemeSpec(.regexLiteralPattern, text: "\na\nb\n"), + LexemeSpec(.regexSlash, text: "/"), + LexemeSpec(.extendedRegexDelimiter, text: "#"), ] ) assertLexemes( "#/ \na\nb\n /#", lexemes: [ - LexemeSpec(.regexLiteral, text: "#/ \na\nb\n /#") + LexemeSpec(.extendedRegexDelimiter, text: "#"), + LexemeSpec(.regexSlash, text: "/"), + LexemeSpec(.regexLiteralPattern, text: " \na\nb\n "), + LexemeSpec(.regexSlash, text: "/"), + LexemeSpec(.extendedRegexDelimiter, text: "#"), ] ) assertLexemes( "##/ \na\nb\n /##", lexemes: [ - LexemeSpec(.regexLiteral, text: "##/ \na\nb\n /##") + LexemeSpec(.extendedRegexDelimiter, text: "##"), + LexemeSpec(.regexSlash, text: "/"), + LexemeSpec(.regexLiteralPattern, text: " \na\nb\n "), + LexemeSpec(.regexSlash, text: "/"), + LexemeSpec(.extendedRegexDelimiter, text: "##"), ] ) assertLexemes( "#/abc/def/#", lexemes: [ - LexemeSpec(.regexLiteral, text: "#/abc/def/#") + LexemeSpec(.extendedRegexDelimiter, text: "#"), + LexemeSpec(.regexSlash, text: "/"), + LexemeSpec(.regexLiteralPattern, text: "abc/def"), + LexemeSpec(.regexSlash, text: "/"), + LexemeSpec(.extendedRegexDelimiter, text: "#"), ] ) assertLexemes( "#/abc\\/#def/#", lexemes: [ - LexemeSpec(.regexLiteral, text: "#/abc\\/#def/#") + LexemeSpec(.extendedRegexDelimiter, text: "#"), + LexemeSpec(.regexSlash, text: "/"), + LexemeSpec(.regexLiteralPattern, text: "abc\\/#def"), + LexemeSpec(.regexSlash, text: "/"), + LexemeSpec(.extendedRegexDelimiter, text: "#"), ] ) assertLexemes( "#/abc|#def/#", lexemes: [ - LexemeSpec(.regexLiteral, text: "#/abc|#def/#") + LexemeSpec(.extendedRegexDelimiter, text: "#"), + LexemeSpec(.regexSlash, text: "/"), + LexemeSpec(.regexLiteralPattern, text: "abc|#def"), + LexemeSpec(.regexSlash, text: "/"), + LexemeSpec(.extendedRegexDelimiter, text: "#"), ] ) assertLexemes( "#/abc\n/#", lexemes: [ - LexemeSpec(.pound, text: "#"), - LexemeSpec(.binaryOperator, text: "/"), - LexemeSpec(.identifier, text: "abc"), + LexemeSpec(.extendedRegexDelimiter, text: "#"), + LexemeSpec(.regexSlash, text: "/"), + LexemeSpec(.regexLiteralPattern, text: "abc"), LexemeSpec(.prefixOperator, leading: "\n", text: "/", flags: [.isAtStartOfLine]), LexemeSpec(.pound, text: "#"), ] @@ -406,9 +440,9 @@ public class LexerTests: XCTestCase { assertLexemes( "#/abc\r/#", lexemes: [ - LexemeSpec(.pound, text: "#"), - LexemeSpec(.binaryOperator, text: "/"), - LexemeSpec(.identifier, text: "abc"), + LexemeSpec(.extendedRegexDelimiter, text: "#"), + LexemeSpec(.regexSlash, text: "/"), + LexemeSpec(.regexLiteralPattern, text: "abc"), LexemeSpec(.prefixOperator, leading: "\r", text: "/", flags: [.isAtStartOfLine]), LexemeSpec(.pound, text: "#"), ] @@ -422,6 +456,117 @@ public class LexerTests: XCTestCase { LexemeSpec(.postfixOperator, text: "/"), ] ) + // Make sure if we have an unterminated extended literal, we stop at the + // first newline + assertLexemes( + """ + #/ + abc + """, + lexemes: [ + LexemeSpec(.extendedRegexDelimiter, text: "#"), + LexemeSpec(.regexSlash, text: "/"), + LexemeSpec(.regexLiteralPattern, text: ""), + LexemeSpec(.identifier, leading: "\n", text: "abc", flags: .isAtStartOfLine), + ] + ) + // Check that we can split operators okay. + assertLexemes( + "!/abc/", + lexemes: [ + LexemeSpec(.prefixOperator, text: "!"), + LexemeSpec(.regexSlash, text: "/"), + LexemeSpec(.regexLiteralPattern, text: "abc"), + LexemeSpec(.regexSlash, text: "/"), + ] + ) + assertLexemes( + "!^/abc/", + lexemes: [ + LexemeSpec(.prefixOperator, text: "!^"), + LexemeSpec(.regexSlash, text: "/"), + LexemeSpec(.regexLiteralPattern, text: "abc"), + LexemeSpec(.regexSlash, text: "/"), + ] + ) + assertLexemes( + "!#/abc/#", + lexemes: [ + LexemeSpec(.prefixOperator, text: "!"), + LexemeSpec(.extendedRegexDelimiter, text: "#"), + LexemeSpec(.regexSlash, text: "/"), + LexemeSpec(.regexLiteralPattern, text: "abc"), + LexemeSpec(.regexSlash, text: "/"), + LexemeSpec(.extendedRegexDelimiter, text: "#"), + ] + ) + // Make sure we don't lex this as a regex. + assertLexemes( + """ + func /^ () { y/ } + """, + lexemes: [ + LexemeSpec(.keyword, text: "func", trailing: " "), + LexemeSpec(.binaryOperator, text: "/^", trailing: " "), + LexemeSpec(.leftParen, text: "("), + LexemeSpec(.rightParen, text: ")", trailing: " "), + LexemeSpec(.leftBrace, text: "{", trailing: " "), + LexemeSpec(.identifier, text: "y"), + LexemeSpec(.postfixOperator, text: "/", trailing: " "), + LexemeSpec(.rightBrace, text: "}"), + ] + ) + assertLexemes( + "^^/!*/", + lexemes: [ + LexemeSpec(.prefixOperator, text: "^^"), + LexemeSpec(.regexSlash, text: "/"), + LexemeSpec(.regexLiteralPattern, text: "!*"), + LexemeSpec(.regexSlash, text: "/"), + ] + ) + assertLexemes( + "/!*/", + lexemes: [ + LexemeSpec(.regexSlash, text: "/"), + LexemeSpec(.regexLiteralPattern, text: "!*"), + LexemeSpec(.regexSlash, text: "/"), + ] + ) + + // Regex literal, even though it's in operator position. + assertLexemes( + "x /y/", + lexemes: [ + LexemeSpec(.identifier, text: "x", trailing: " "), + LexemeSpec(.regexSlash, text: "/"), + LexemeSpec(.regexLiteralPattern, text: "y"), + LexemeSpec(.regexSlash, text: "/"), + ] + ) + assertLexemes( + "x /.^ y/", + lexemes: [ + LexemeSpec(.identifier, text: "x", trailing: " "), + LexemeSpec(.regexSlash, text: "/"), + LexemeSpec(.regexLiteralPattern, text: ".^ y"), + LexemeSpec(.regexSlash, text: "/"), + ] + ) + + // Comments, not regex literals + assertLexemes( + "^//", + lexemes: [ + LexemeSpec(.binaryOperator, text: "^", trailing: "//") + ] + ) + assertLexemes( + "^/*/", + lexemes: [ + LexemeSpec(.binaryOperator, text: "^", trailing: "/*/") + ] + ) } func testUnexpectedLexing() { @@ -613,13 +758,29 @@ public class LexerTests: XCTestCase { assertLexemes( "#/abc|#def/", lexemes: [ - LexemeSpec(.pound, text: "#"), - LexemeSpec(.binaryOperator, text: "/"), - LexemeSpec(.identifier, text: "abc"), - LexemeSpec(.binaryOperator, text: "|"), - LexemeSpec(.pound, text: "#"), - LexemeSpec(.identifier, text: "def"), - LexemeSpec(.postfixOperator, text: "/"), + LexemeSpec(.extendedRegexDelimiter, text: "#"), + LexemeSpec(.regexSlash, text: "/"), + LexemeSpec(.regexLiteralPattern, text: "abc|#def"), + LexemeSpec(.regexSlash, text: "/"), + ] + ) + + assertLexemes( + "#/abc|#def//", + lexemes: [ + LexemeSpec(.extendedRegexDelimiter, text: "#"), + LexemeSpec(.regexSlash, text: "/"), + LexemeSpec(.regexLiteralPattern, text: "abc|#def/"), + LexemeSpec(.regexSlash, text: "/"), + ] + ) + // This is an operator, not a regex. + assertLexemes( + "lhs /==/ rhs", + lexemes: [ + LexemeSpec(.identifier, text: "lhs", trailing: " "), + LexemeSpec(.binaryOperator, text: "/==/", trailing: " "), + LexemeSpec(.identifier, text: "rhs"), ] ) } @@ -903,6 +1064,24 @@ public class LexerTests: XCTestCase { } } + func testInvalidUTF8RegexLiteral() { + let slashByte = UInt8(UnicodeScalar("/").value) + let sourceBytes: [UInt8] = [slashByte, 0xfd, slashByte] + + lex(sourceBytes) { lexemes in + guard lexemes.count == 4 else { + return XCTFail("Expected 4 lexemes, got \(lexemes.count)") + } + assertRawBytesLexeme( + lexemes[1], + kind: .regexLiteralPattern, + leadingTrivia: [], + text: [0xfd], + error: TokenDiagnostic(.invalidUtf8, byteOffset: 1) + ) + } + } + func testInterpolatedString() { assertLexemes( #""" diff --git a/Tests/SwiftParserTest/RegexLiteralTests.swift b/Tests/SwiftParserTest/RegexLiteralTests.swift new file mode 100644 index 00000000000..e67c67f8e72 --- /dev/null +++ b/Tests/SwiftParserTest/RegexLiteralTests.swift @@ -0,0 +1,826 @@ +@_spi(RawSyntax) import SwiftSyntax +@_spi(RawSyntax) import SwiftParser +import XCTest + +final class RegexLiteralTests: XCTestCase { + func testForwardSlash1() { + assertParse( + #""" + /(?[[:alpha:]]\w*) = (?[0-9A-F]+)/ + """# + ) + } + func testForwardSlash2() { + assertParse( + """ + postfix func /(lhs: Int) -> Int {1/} + """ + ) + } + + func testEmpty() { + assertParse("#//#") + } + + func testExtraneous1() { + assertParse( + #""" + #//#1️⃣# + """#, + diagnostics: [ + DiagnosticSpec(locationMarker: "1️⃣", message: "too many '#' characters in closing delimiter") + ], + fixedSource: "#//#" + ) + } + func testExtraneous2() { + assertParse( + """ + #/abc/#1️⃣# + """, + diagnostics: [ + DiagnosticSpec(locationMarker: "1️⃣", message: "too many '#' characters in closing delimiter") + ], + fixedSource: "#/abc/#" + ) + } + + func testUnterminated1() { + assertParse( + #""" + #1️⃣ + """#, + diagnostics: [ + DiagnosticSpec(message: "expected identifier in macro expansion") + ] + ) + } + func testUnterminated2() { + assertParse( + #""" + 1️⃣/ + """#, + diagnostics: [ + DiagnosticSpec(message: "extraneous code '/' at top level") + ] + ) + } + func testUnterminated3() { + assertParse( + #""" + /#1️⃣ + """#, + diagnostics: [ + DiagnosticSpec(message: "expected identifier in macro expansion") + ] + ) + } + func testUnterminated4() { + assertParse( + #""" + #/1️⃣ + """#, + diagnostics: [ + DiagnosticSpec(message: "expected '/#' to end regex literal") + ] + ) + } + func testUnterminated5() { + assertParse( + #""" + #//1️⃣ + """#, + diagnostics: [ + DiagnosticSpec(message: "expected '#' to end regex literal") + ] + ) + } + func testUnterminated6() { + assertParse( + #""" + #///1️⃣ + """#, + diagnostics: [ + DiagnosticSpec(message: "expected '#' to end regex literal") + ] + ) + } + func testUnterminated7() { + assertParse( + #""" + #/#1️⃣ + """#, + diagnostics: [ + DiagnosticSpec(message: "expected '/#' to end regex literal") + ] + ) + } + func testUnterminated8() { + assertParse( + #""" + #/##1️⃣ + """#, + diagnostics: [ + DiagnosticSpec(message: "expected '/#' to end regex literal") + ] + ) + } + func testUnterminated9() { + assertParse( + #""" + #/##/1️⃣ + """#, + diagnostics: [ + DiagnosticSpec(message: "expected '#' to end regex literal") + ] + ) + } + func testUnterminated10() { + assertParse( + #""" + ##/##/#1️⃣ + """#, + diagnostics: [ + DiagnosticSpec(message: "expected additional '#' characters in closing delimiter") + ], + fixedSource: "##/##/##" + ) + } + func testUnterminated11() { + assertParse( + #""" + ##/###1️⃣ + """#, + diagnostics: [ + DiagnosticSpec(message: "expected '/##' to end regex literal") + ], + fixedSource: "##/###/##" + ) + } + func testUnterminated12() { + assertParse( + #""" + #/\/#1️⃣ + """#, + diagnostics: [ + DiagnosticSpec(message: "expected '/#' to end regex literal") + ], + fixedSource: #"#/\/#/#"# + ) + } + func testUnterminated13() { + assertParse( + #""" + ##/abc/#def1️⃣ + """#, + diagnostics: [ + DiagnosticSpec(message: "expected '/##' to end regex literal") + ], + fixedSource: "##/abc/#def/##" + ) + } + func testUnterminated14() { + assertParse( + #""" + ##/abc/def#1️⃣ + """#, + diagnostics: [ + DiagnosticSpec(message: "expected '/##' to end regex literal") + ], + fixedSource: "##/abc/def#/##" + ) + } + + func testTerminated1() { + assertParse( + #""" + #//# + """# + ) + } + func testTerminated2() { + assertParse( + #""" + #///# + """# + ) + } + func testTerminated3() { + assertParse( + #""" + #/#//# + """# + ) + } + func testTerminated4() { + assertParse( + #""" + ##/##/## + """# + ) + } + func testTerminated5() { + assertParse( + #""" + #/\/#/# + """# + ) + } + func testTerminated6() { + assertParse( + #""" + #/\//# + """# + ) + } + func testTerminated7() { + assertParse( + #""" + #/\\/# + """# + ) + } + + func testUnprintable1() { + assertParse( + """ + /1️⃣\u{7F}/ + """, + diagnostics: [ + DiagnosticSpec(message: "unprintable ASCII character found in source file") + ] + ) + } + func testUnprintable2() { + assertParse( + """ + #/1️⃣\u{7F}/# + """, + diagnostics: [ + DiagnosticSpec(message: "unprintable ASCII character found in source file") + ] + ) + } + + func testMultiline1() { + assertParse( + """ + #/ + abc1️⃣/# + """, + diagnostics: [ + DiagnosticSpec(message: "multi-line regex closing delimiter must appear on new line") + ] + ) + } + func testMultiline2() { + assertParse( + """ + #/abc1️⃣ + /#2️⃣ + """, + diagnostics: [ + DiagnosticSpec(locationMarker: "1️⃣", message: "expected '/#' to end regex literal"), + DiagnosticSpec(locationMarker: "2️⃣", message: "expected identifier in macro expansion"), + ] + ) + } + func testMultiline3() { + assertParse( + """ + #/abc1️⃣ + \t \t /#2️⃣ + """, + diagnostics: [ + DiagnosticSpec(locationMarker: "1️⃣", message: "expected '/#' to end regex literal"), + DiagnosticSpec(locationMarker: "2️⃣", message: "expected identifier in macro expansion"), + ] + ) + } + func testMultiline4() { + assertParse( + """ + #/ + abc + \t \t /# + """ + ) + } + func testMultiline5() { + assertParse( + """ + #/ + #1️⃣/# + """, + diagnostics: [ + DiagnosticSpec(message: "multi-line regex closing delimiter must appear on new line") + ] + ) + } + func testOpeningSpace1() { + assertParse( + """ + 1️⃣/ a/ + """, + diagnostics: [ + DiagnosticSpec(message: "extraneous code '/ a/' at top level") + ] + ) + } + func testOpeningSpace2() { + assertParse( + """ + let x = /1️⃣ a/ + """, + diagnostics: [ + DiagnosticSpec(message: "bare slash regex literal may not start with space") + ] + ) + } + func testOpeningSpace3() { + assertParse( + """ + #/ a/# + """ + ) + } + func testClosingSpace1() { + assertParse( + """ + /a /1️⃣ + """, + diagnostics: [ + DiagnosticSpec(message: "expected expression after operator") + ] + ) + } + func testClosingSpace2() { + assertParse( + """ + let x = /a /1️⃣ + """, + diagnostics: [ + DiagnosticSpec(message: "expected expression after operator") + ] + ) + } + func testClosingSpace3() { + assertParse( + """ + #/a /# + """ + ) + } + func testOpeningAndClosingSpace1() { + assertParse( + """ + 1️⃣/ / + """, + diagnostics: [ + DiagnosticSpec(message: "extraneous code '/ /' at top level") + ] + ) + } + func testOpeningAndClosingSpace2() { + assertParse( + """ + x += / 1️⃣ / + """, + diagnostics: [ + DiagnosticSpec(message: "bare slash regex literal may not end with space") + ] + ) + } + func testOpeningAndClosingSpace3() { + assertParse( + """ + #/ /# + """ + ) + } + func testOpeningAndClosingSpace4() { + assertParse( + """ + 1️⃣/ / + """, + diagnostics: [ + DiagnosticSpec(message: "extraneous code '/ /' at top level") + ] + ) + } + func testOpeningAndClosingSpace5() { + assertParse( + """ + let x = /1️⃣ / + """, + diagnostics: [ + DiagnosticSpec(message: "bare slash regex literal may not end with space") + ] + ) + } + func testOpeningAndClosingSpace6() { + assertParse( + """ + #/ /# + """ + ) + } + func testSingleLineTabChar() { + // We currently only keep track of one lexer error, so only diagnose the second. + assertParse( + """ + #/\t1️⃣\t/# + """, + diagnostics: [ + DiagnosticSpec(message: "unprintable ASCII character found in source file") + ] + ) + } + + func testBinOpDisambiguation1() { + assertParse( + """ + x /^ y ^/ z + """ + ) + } + func testBinOpDisambiguation2() { + assertParse( + """ + x /^ y/ + """ + ) + } + func testBinOpDisambiguation3() { + assertParse( + """ + x !/^ y/ + """ + ) + } + func testBinOpDisambiguation4() { + assertParse( + """ + x !/^ y !/ z + """ + ) + } + func testBinOpDisambiguation5() { + assertParse( + """ + try? /^ x/ + """ + ) + } + func testBinOpDisambiguation6() { + assertParse( + """ + try? /^ x ^/ + """ + ) + } + func testBinOpDisambiguation7() { + assertParse( + """ + try! /^ x/ + """ + ) + } + func testBinOpDisambiguation8() { + assertParse( + """ + try? /^ x ^/ + """ + ) + } + func testBinOpDisambiguation9() { + assertParse( + """ + x < /^ }}x/ + """ + ) + } + func testBinOpDisambiguation10() { + assertParse( + """ + { /^ }}x/ } + """ + ) + } + func testBinOpDisambiguation11() { + assertParse( + """ + ( /^ }}x/ ) + """ + ) + } + func testBinOpDisambiguation12() { + assertParse( + """ + [ /^ }}x/ ] + """ + ) + } + func testBinOpDisambiguation13() { + assertParse( + """ + foo(&/^ }}x/) + """ + ) + } + func testBinOpDisambiguation14() { + assertParse( + """ + x; /^ }}x/ + """ + ) + } + func testBinOpDisambiguation15() { + assertParse( + """ + [0 : /^ }}x/] + """ + ) + } + func testBinOpDisambiguation16() { + assertParse( + """ + (0, /^ }}x/) + """ + ) + } + func testBinOpDisambiguation17() { + assertParse( + """ + x^ /^ x/ + """ + ) + } + func testBinOpDisambiguation18() { + assertParse( + """ + x! /^ x/ + """ + ) + } + func testBinOpDisambiguation19() { + assertParse( + """ + x? /^ x/ + """ + ) + } + func testBinOpDisambiguation20() { + assertParse( + """ + x > /^ }}x/ + """ + ) + } + func testBinOpDisambiguation21() { + assertParse( + """ + {} /^ x/ + """ + ) + } + func testBinOpDisambiguation22() { + assertParse( + """ + () /^ x/ + """ + ) + } + func testBinOpDisambiguation23() { + assertParse( + """ + [] /^ x/ + """ + ) + } + func testBinOpDisambiguation24() { + assertParse( + """ + x... /^ x/ + """ + ) + } + func testBinOpDisambiguation25() { + assertParse( + """ + x.1️⃣ /^ x/ + """, + diagnostics: [ + DiagnosticSpec(message: "expected name in member access") + ] + ) + } + func testBinOpDisambiguation26() { + // FIXME: The diagnostic should be one character back + assertParse( + """ + # 1️⃣/^ x/ + """, + diagnostics: [ + DiagnosticSpec(message: "expected identifier in macro expansion") + ] + ) + } + func testBinOpDisambiguation27() { + assertParse( + """ + `x` /^ x/ + """ + ) + } + func testBinOpDisambiguation28() { + // FIXME: The diagnostic should be one character back + assertParse( + #""" + \ 1️⃣/^ x/ + """#, + diagnostics: [ + DiagnosticSpec(message: "expected root in key path") + ] + ) + } + func testBinOpDisambiguation29() { + assertParse( + """ + x /^ x/ + """ + ) + } + func testBinOpDisambiguation30() { + assertParse( + """ + true /^ x/ + """ + ) + } + func testBinOpDisambiguation31() { + assertParse( + """ + false /^ x/ + """ + ) + } + func testBinOpDisambiguation32() { + assertParse( + """ + try /^ }}x/ + """ + ) + } + func testBinOpDisambiguation33() { + assertParse( + """ + x as Any /^ x/ + """ + ) + } + func testBinOpDisambiguation34() { + assertParse( + """ + nil /^ x/ + """ + ) + } + func testBinOpDisambiguation35() { + assertParse( + """ + .none /^ x/ + """ + ) + } + func testBinOpDisambiguation36() { + assertParse( + """ + .objc /^ x/ + """ + ) + } + func testBinOpDisambiguation37() { + assertParse( + """ + P.Protocol /^ x/ + """ + ) + } + func testBinOpDisambiguation38() { + assertParse( + """ + self /^ x/ + """ + ) + } + func testBinOpDisambiguation39() { + assertParse( + """ + Self /^ x/ + """ + ) + } + func testBinOpDisambiguation40() { + assertParse( + """ + super /^ x/ + """ + ) + } + func testBinOpDisambiguation41() { + // await is a contextual keyword, so we can't assume it must be a regex. + assertParse( + """ + await 1️⃣/^ x/ + """, + diagnostics: [ + DiagnosticSpec(message: "expected expression in 'await' expression") + ] + ) + } + func testBinOpDisambiguation42() { + // await is a contextual keyword, so we can't assume it must be a regex. + assertParse( + """ + ^await /^ x/ + """ + ) + } + func testBinOpDisambiguation43() { + assertParse( + """ + x ? /^ }}x/ : /x/ + """ + ) + } + func testBinOpDisambiguation44() { + assertParse( + """ + x ? /x/ : /^ }}x/ + """ + ) + } + func testBinOpDisambiguation45() { + assertParse( + """ + 0 /^ x/ + """ + ) + } + func testBinOpDisambiguation46() { + assertParse( + """ + 0.1 /^ x/ + """ + ) + } + func testBinOpDisambiguation47() { + assertParse( + """ + #if /^ }}x/ + #endif + """ + ) + } + func testBinOpDisambiguation48() { + assertParse( + """ + #if true + #else + /^ }}x/ + #endif + """ + ) + } + func testBinOpDisambiguation49() { + assertParse( + """ + #if true + #elseif /^ }}x/ + #endif + """ + ) + } + func testBinOpDisambiguation50() { + assertParse( + """ + #if true + #endif + /^ }}x/ + """ + ) + } + + func testNulCharacter() { + assertParse( + "/1️⃣\0/", + diagnostics: [ + DiagnosticSpec(message: "nul character embedded in middle of file", severity: .warning) + ] + ) + } + func testEmoji() { + assertParse("/👍/") + } +} diff --git a/Tests/SwiftParserTest/translated/ForwardSlashRegexSkippingInvalidTests.swift b/Tests/SwiftParserTest/translated/ForwardSlashRegexSkippingInvalidTests.swift index 31f9577e7aa..74dc16f52ec 100644 --- a/Tests/SwiftParserTest/translated/ForwardSlashRegexSkippingInvalidTests.swift +++ b/Tests/SwiftParserTest/translated/ForwardSlashRegexSkippingInvalidTests.swift @@ -8,12 +8,10 @@ final class ForwardSlashRegexSkippingInvalidTests: XCTestCase { """ // We don't consider this a regex literal when skipping as it has an initial // space. - func a() { _ = 1️⃣/ x*/ } + func a() { _ = /1️⃣ x*/ } """, diagnostics: [ - // TODO: Old parser expected error on line 3: unexpected end of block comment - DiagnosticSpec(message: "expected expression in function"), - DiagnosticSpec(message: "unexpected code '/ x*/' in function"), + DiagnosticSpec(message: "bare slash regex literal may not start with space") ] ) } @@ -25,8 +23,7 @@ final class ForwardSlashRegexSkippingInvalidTests: XCTestCase { func b() { _ = /x1️⃣)*/ } """, diagnostics: [ - // TODO: Old parser expected error on line 2: unexpected end of block comment - DiagnosticSpec(message: "unexpected code ')*/' in function"), + DiagnosticSpec(message: "unexpected code ')*/' in function") ] ) } @@ -35,18 +32,12 @@ final class ForwardSlashRegexSkippingInvalidTests: XCTestCase { assertParse( """ // These also fail the heuristic, but have unbalanced `{` `}`, so we don't skip. - func c() { _ = 1️⃣/ x}2️⃣*/ } - func d() { _ = 3️⃣/ x{*/ } + func c() { _ = /1️⃣ x}*/ } + func d() { _ = /2️⃣ x{*/ } """, diagnostics: [ - // TODO: Old parser expected error on line 2: regex literal may not start with space; add backslash to escape - DiagnosticSpec(locationMarker: "1️⃣", message: "expected expression in function"), - DiagnosticSpec(locationMarker: "1️⃣", message: "unexpected code '/ x' in function"), - DiagnosticSpec(locationMarker: "2️⃣", message: "unexpected code '*/ }' before function"), - // TODO: Old parser expected error on line 3: regex literal may not start with space; add backslash to escape - DiagnosticSpec(locationMarker: "3️⃣", message: "expected expression in function"), - DiagnosticSpec(locationMarker: "3️⃣", message: "expected '}' to end function"), - DiagnosticSpec(locationMarker: "3️⃣", message: "extraneous code '/ x{*/ }' at top level"), + DiagnosticSpec(locationMarker: "1️⃣", message: "bare slash regex literal may not start with space"), + DiagnosticSpec(locationMarker: "2️⃣", message: "bare slash regex literal may not start with space"), ] ) } @@ -56,15 +47,12 @@ final class ForwardSlashRegexSkippingInvalidTests: XCTestCase { """ // Unterminated, and unbalanced `{}`. func e() { - _ = 1️⃣/ } - 2️⃣} + _ = /1️⃣ }2️⃣ + } """, diagnostics: [ - DiagnosticSpec(locationMarker: "1️⃣", message: "expected expression in function"), - DiagnosticSpec(locationMarker: "1️⃣", message: "unexpected code '/' in function"), - DiagnosticSpec(locationMarker: "2️⃣", message: "extraneous brace at top level"), - // TODO: Old parser expected error on line 0: unterminated regex literal - // TODO: Old parser expected error on line 0: regex literal may not start with space; add backslash to escape + DiagnosticSpec(locationMarker: "1️⃣", message: "bare slash regex literal may not start with space"), + DiagnosticSpec(locationMarker: "2️⃣", message: "expected '/' to end regex literal"), ] ) } @@ -73,15 +61,12 @@ final class ForwardSlashRegexSkippingInvalidTests: XCTestCase { assertParse( """ func f() { - _ = 1️⃣/ { + _ = /1️⃣ {2️⃣ } """, diagnostics: [ - // TODO: Old parser expected error on line 3: unterminated regex literal - // TODO: Old parser expected error on line 3: regex literal may not start with space; add backslash to escape - DiagnosticSpec(message: "expected expression in function"), - DiagnosticSpec(message: "expected '}' to end function"), - DiagnosticSpec(message: "extraneous code at top level"), + DiagnosticSpec(locationMarker: "1️⃣", message: "bare slash regex literal may not start with space"), + DiagnosticSpec(locationMarker: "2️⃣", message: "expected '/' to end regex literal"), ] ) } @@ -94,8 +79,7 @@ final class ForwardSlashRegexSkippingInvalidTests: XCTestCase { 1️⃣} """, diagnostics: [ - // TODO: Old parser expected error on line 1: extraneous '}' at top level - DiagnosticSpec(message: "extraneous brace at top level"), + DiagnosticSpec(message: "extraneous brace at top level") ] ) } @@ -115,12 +99,12 @@ final class ForwardSlashRegexSkippingInvalidTests: XCTestCase { assertParse( #""" func i() { - _ = /x 2️⃣"[abc] { + _ = /x1️⃣ "[abc] {2️⃣ } """#, diagnostics: [ - // TODO: Old parser expected error on line 2: unterminated string literal - DiagnosticSpec(locationMarker: "2️⃣", message: #"unexpected code '"[abc] {' in function"#), + DiagnosticSpec(locationMarker: "1️⃣", message: "consecutive statements on a line must be separated by ';'"), + DiagnosticSpec(locationMarker: "2️⃣", message: #"expected '"' to end string literal"#), ] ) } @@ -129,14 +113,11 @@ final class ForwardSlashRegexSkippingInvalidTests: XCTestCase { assertParse( """ func j() { - _ = 1️⃣/^ [abc] { + _ = /^ [abc] {1️⃣ } """, diagnostics: [ - // TODO: Old parser expected error on line 2: unterminated regex literal - DiagnosticSpec(message: "expected expression in function"), - DiagnosticSpec(message: "expected '}' to end function"), - DiagnosticSpec(message: "extraneous code at top level"), + DiagnosticSpec(message: "expected '/' to end regex literal") ] ) } @@ -145,13 +126,11 @@ final class ForwardSlashRegexSkippingInvalidTests: XCTestCase { assertParse( #""" func k() { - _ = 1️⃣/^ "[abc] { + _ = /^ "[abc] {1️⃣ } """#, diagnostics: [ - // TODO: Old parser expected error on line 2: unterminated string literal - DiagnosticSpec(message: "expected expression in function"), - DiagnosticSpec(message: #"unexpected code '/^ "[abc] {' in function"#), + DiagnosticSpec(message: "expected '/' to end regex literal") ] ) } @@ -160,13 +139,11 @@ final class ForwardSlashRegexSkippingInvalidTests: XCTestCase { assertParse( """ func l() { - _ = 1️⃣/^ } abc { + _ = /^ } abc {1️⃣ } """, diagnostics: [ - // TODO: Old parser expected error on line 2: unterminated regex literal - DiagnosticSpec(message: "expected expression in function"), - DiagnosticSpec(message: "unexpected code '/^' in function"), + DiagnosticSpec(message: "expected '/' to end regex literal") ] ) } @@ -175,16 +152,14 @@ final class ForwardSlashRegexSkippingInvalidTests: XCTestCase { assertParse( #""" func m() { - _ = 1️⃣/ " - } - 2️⃣} + _ = /1️⃣ "2️⃣ + } + 3️⃣} """#, diagnostics: [ - // TODO: Old parser expected error on line 2: unterminated string literal - DiagnosticSpec(locationMarker: "1️⃣", message: "expected expression in function"), - DiagnosticSpec(locationMarker: "1️⃣", message: #"unexpected code '/ "' in function"#), - // TODO: Old parser expected error on line 4: extraneous '}' at top level - DiagnosticSpec(locationMarker: "2️⃣", message: "extraneous brace at top level"), + DiagnosticSpec(locationMarker: "1️⃣", message: "bare slash regex literal may not start with space"), + DiagnosticSpec(locationMarker: "2️⃣", message: "expected '/' to end regex literal"), + DiagnosticSpec(locationMarker: "3️⃣", message: "extraneous brace at top level"), ] ) @@ -250,13 +225,10 @@ final class ForwardSlashRegexSkippingInvalidTests: XCTestCase { func testForwardSlashRegexSkippingInvalid17() { assertParse( """ - func err1() { _ = 1️⃣/ 0xG}2️⃣/ } + func err1() { _ = /1️⃣ 0xG}/ } """, diagnostics: [ - // TODO: Old parser expected error on line 1: regex literal may not start with space; add backslash to escape - DiagnosticSpec(locationMarker: "1️⃣", message: "expected expression in function"), - DiagnosticSpec(locationMarker: "1️⃣", message: "unexpected code '/ 0xG' in function"), - DiagnosticSpec(locationMarker: "2️⃣", message: "extraneous code '/ }' at top level"), + DiagnosticSpec(message: "bare slash regex literal may not start with space") ] ) } @@ -264,27 +236,21 @@ final class ForwardSlashRegexSkippingInvalidTests: XCTestCase { func testForwardSlashRegexSkippingInvalid18() { assertParse( """ - func err2() { _ = 1️⃣/ 0oG}2️⃣/ } + func err2() { _ = /1️⃣ 0oG}/ } """, diagnostics: [ - // TODO: Old parser expected error on line 1: regex literal may not start with space; add backslash to escape - DiagnosticSpec(locationMarker: "1️⃣", message: "expected expression in function"), - DiagnosticSpec(locationMarker: "1️⃣", message: "unexpected code '/ 0oG' in function"), - DiagnosticSpec(locationMarker: "2️⃣", message: "extraneous code '/ }' at top level"), + DiagnosticSpec(message: "bare slash regex literal may not start with space") ] ) } func testForwardSlashRegexSkippingInvalid19() { assertParse( - #""" - func err3() { _ = 1️⃣/ {"/ } - """#, + """ + func err3() { _ = /1️⃣ {"/ } + """, diagnostics: [ - // TODO: Old parser expected error on line 1: regex literal may not start with space; add backslash to escape - DiagnosticSpec(message: "expected expression in function"), - DiagnosticSpec(message: "expected '}' to end function"), - DiagnosticSpec(message: #"extraneous code '/ {"/ }' at top level"#), + DiagnosticSpec(message: "bare slash regex literal may not start with space") ] ) } @@ -292,13 +258,10 @@ final class ForwardSlashRegexSkippingInvalidTests: XCTestCase { func testForwardSlashRegexSkippingInvalid20() { assertParse( """ - func err4() { _ = 1️⃣/ {'/ } + func err4() { _ = /1️⃣ {'/ } """, diagnostics: [ - // TODO: Old parser expected error on line 1: regex literal may not start with space; add backslash to escape - DiagnosticSpec(message: "expected expression in function"), - DiagnosticSpec(message: "expected '}' to end function"), - DiagnosticSpec(message: "extraneous code '/ {'/ }' at top level"), + DiagnosticSpec(message: "bare slash regex literal may not start with space") ] ) } @@ -306,13 +269,10 @@ final class ForwardSlashRegexSkippingInvalidTests: XCTestCase { func testForwardSlashRegexSkippingInvalid21() { assertParse( """ - func err5() { _ = 1️⃣/ {<#placeholder#>/ } + func err5() { _ = /1️⃣ {<#placeholder#>/ } """, diagnostics: [ - // TODO: Old parser expected error on line 1: regex literal may not start with space; add backslash to escape - DiagnosticSpec(message: "expected expression in function"), - DiagnosticSpec(message: "expected '}' to end function"), - DiagnosticSpec(message: "extraneous code '/ {<#placeholder#>/ }' at top level"), + DiagnosticSpec(message: "bare slash regex literal may not start with space") ] ) } diff --git a/Tests/SwiftParserTest/translated/ForwardSlashRegexSkippingTests.swift b/Tests/SwiftParserTest/translated/ForwardSlashRegexSkippingTests.swift index 9933da12055..a8384cee2eb 100644 --- a/Tests/SwiftParserTest/translated/ForwardSlashRegexSkippingTests.swift +++ b/Tests/SwiftParserTest/translated/ForwardSlashRegexSkippingTests.swift @@ -77,11 +77,7 @@ final class ForwardSlashRegexSkippingTests: XCTestCase { assertParse( """ func d() { _ = ^^/x}1️⃣}*/ } - """, - diagnostics: [ - // FIXME: This should not emit any diagnostics. - DiagnosticSpec(message: "extraneous code '}*/ }' at top level"), - ] + """ ) } @@ -89,12 +85,7 @@ final class ForwardSlashRegexSkippingTests: XCTestCase { assertParse( """ func e() { _ = (^^/x1️⃣}2️⃣}*/) } - """, - diagnostics: [ - // FIXME: This should not emit any diagnostics. - DiagnosticSpec(locationMarker: "1️⃣", message: "expected ')' to end tuple"), - DiagnosticSpec(locationMarker: "2️⃣", message: "extraneous code '}*/) }' at top level"), - ] + """ ) } @@ -102,11 +93,7 @@ final class ForwardSlashRegexSkippingTests: XCTestCase { assertParse( """ func f() { _ = ^^/^x}1️⃣}*/ } - """, - diagnostics: [ - // FIXME: This should not emit any diagnostics. - DiagnosticSpec(message: "extraneous code '}*/ }' at top level"), - ] + """ ) } @@ -122,11 +109,7 @@ final class ForwardSlashRegexSkippingTests: XCTestCase { assertParse( #""" func h() { _ = "\(^^/x1️⃣}}*/)" } - """#, - diagnostics: [ - // FIXME: This should not emit any diagnostics. - DiagnosticSpec(message: "unexpected code '}}*/' in string literal"), - ] + """# ) } @@ -322,26 +305,19 @@ final class ForwardSlashRegexSkippingTests: XCTestCase { func testForwardSlashRegexSkipping37() { assertParse( """ - func a3() { _ = 1️⃣/)/ } - """, - diagnostics: [ - // FIXME: Improve diagnostics (ideally we'd be able to parse as an - // invalid regex) - DiagnosticSpec(message: "expected expression in function"), - DiagnosticSpec(message: "unexpected code '/)/' in function"), - ] + func a3() { _ = /)/ } + """ ) } func testForwardSlashRegexSkipping38() { assertParse( """ - func a4() { _ = 1️⃣/ / } + func a4() { _ = /1️⃣ / } """, diagnostics: [ - // TODO: Old parser expected error on line 1: regex literal may not start with space; add backslash to escape - DiagnosticSpec(message: "expected expression in function"), - DiagnosticSpec(message: "unexpected code '/ /' in function"), + // TODO: Old parser had a fix-it to add backslash to escape + DiagnosticSpec(message: "bare slash regex literal may not end with space") ] ) } @@ -407,12 +383,7 @@ final class ForwardSlashRegexSkippingTests: XCTestCase { assertParse( """ func err6() { _ = ^^/1️⃣0xG/ } - """, - diagnostics: [ - // FIXME: This should not emit any diagnostics. - DiagnosticSpec(message: "expected expression in prefix operator expression"), - DiagnosticSpec(message: "unexpected code '0xG/' in function"), - ] + """ ) } @@ -420,12 +391,7 @@ final class ForwardSlashRegexSkippingTests: XCTestCase { assertParse( """ func err7() { _ = ^^/1️⃣0oG/ } - """, - diagnostics: [ - // FIXME: This should not emit any diagnostics. - DiagnosticSpec(message: "expected expression in prefix operator expression"), - DiagnosticSpec(message: "unexpected code '0oG/' in function"), - ] + """ ) } @@ -433,12 +399,7 @@ final class ForwardSlashRegexSkippingTests: XCTestCase { assertParse( #""" func err8() { _ = ^^/"/ }1️⃣ - """#, - diagnostics: [ - // FIXME: This should not emit any diagnostics. - DiagnosticSpec(message: #"expected '"' to end string literal"#), - DiagnosticSpec(message: "expected '}' to end function"), - ] + """# ) } @@ -446,12 +407,7 @@ final class ForwardSlashRegexSkippingTests: XCTestCase { assertParse( """ func err9() { _ = ^^/'/ }1️⃣ - """, - diagnostics: [ - // FIXME: This should not emit any diagnostics. - DiagnosticSpec(message: "expected ''' in string literal"), - DiagnosticSpec(message: "expected '}' to end function"), - ] + """ ) } @@ -467,12 +423,7 @@ final class ForwardSlashRegexSkippingTests: XCTestCase { assertParse( """ func err11() { _ = (^^/1️⃣0xG/) } - """, - diagnostics: [ - // FIXME: This should not emit any diagnostics. - DiagnosticSpec(message: "expected expression in prefix operator expression"), - DiagnosticSpec(message: "unexpected code '0xG/' in tuple"), - ] + """ ) } @@ -480,12 +431,7 @@ final class ForwardSlashRegexSkippingTests: XCTestCase { assertParse( """ func err12() { _ = (^^/1️⃣0oG/) } - """, - diagnostics: [ - // FIXME: This should not emit any diagnostics. - DiagnosticSpec(message: "expected expression in prefix operator expression"), - DiagnosticSpec(message: "unexpected code '0oG/' in tuple"), - ] + """ ) } @@ -493,13 +439,7 @@ final class ForwardSlashRegexSkippingTests: XCTestCase { assertParse( #""" func err13() { _ = (^^/"/) }1️⃣ - """#, - diagnostics: [ - // FIXME: This should not emit any diagnostics. - DiagnosticSpec(message: #"expected '"' to end string literal"#), - DiagnosticSpec(message: "expected ')' to end tuple"), - DiagnosticSpec(message: "expected '}' to end function"), - ] + """# ) } @@ -507,13 +447,7 @@ final class ForwardSlashRegexSkippingTests: XCTestCase { assertParse( """ func err14() { _ = (^^/'/) }1️⃣ - """, - diagnostics: [ - // FIXME: This should not emit any diagnostics. - DiagnosticSpec(message: "expected ''' in string literal"), - DiagnosticSpec(message: "expected ')' to end tuple"), - DiagnosticSpec(message: "expected '}' to end function"), - ] + """ ) } diff --git a/Tests/SwiftParserTest/translated/ForwardSlashRegexTests.swift b/Tests/SwiftParserTest/translated/ForwardSlashRegexTests.swift index 9e64fd22b5e..d3c1cf704b1 100644 --- a/Tests/SwiftParserTest/translated/ForwardSlashRegexTests.swift +++ b/Tests/SwiftParserTest/translated/ForwardSlashRegexTests.swift @@ -53,12 +53,8 @@ final class ForwardSlashRegexTests: XCTestCase { func testForwardSlashRegex11() { assertParse( """ - let i = 01️⃣ /^/2️⃣ 1/^/3 - """, - diagnostics: [ - DiagnosticSpec(locationMarker: "1️⃣", message: "consecutive statements on a line must be separated by ';'"), - DiagnosticSpec(locationMarker: "2️⃣", message: "consecutive statements on a line must be separated by ';'"), - ] + let i = 0 /^/ 1/^/3 + """ ) } @@ -112,10 +108,7 @@ final class ForwardSlashRegexTests: XCTestCase { } """, diagnostics: [ - // TODO: Old parser expected error on line 4: type annotation missing in pattern - // TODO: Old parser expected error on line 4: consecutive statements on a line must be separated by ';' - // TODO: Old parser expected error on line 4: expected expression - DiagnosticSpec(message: "unexpected code '=/0/' in 'do' statement"), + DiagnosticSpec(message: "unexpected code '=/0/' in 'do' statement") ] ) } @@ -157,13 +150,10 @@ final class ForwardSlashRegexTests: XCTestCase { func testForwardSlashRegex24() { assertParse( """ - _ = 1️⃣!/ / + _ = !/1️⃣ / """, diagnostics: [ - // TODO: Old parser expected error on line 1: regex literal may not start with space; add backslash to escape - - DiagnosticSpec(message: "expected expression"), - DiagnosticSpec(message: "extraneous code '!/ /' at top level"), + DiagnosticSpec(message: "bare slash regex literal may not end with space") ] ) } @@ -171,12 +161,10 @@ final class ForwardSlashRegexTests: XCTestCase { func testForwardSlashRegex25() { assertParse( """ - _ = 1️⃣!!/ / + _ = !!/1️⃣ / """, diagnostics: [ - // TODO: Old parser expected error on line 1: regex literal may not start with space; add backslash to escape - DiagnosticSpec(message: "expected expression"), - DiagnosticSpec(message: "extraneous code '!!/ /' at top level"), + DiagnosticSpec(message: "bare slash regex literal may not end with space") ] ) } @@ -200,12 +188,10 @@ final class ForwardSlashRegexTests: XCTestCase { func testForwardSlashRegex28() { assertParse( """ - _ = 1️⃣/^) + _ = /^)1️⃣ """, diagnostics: [ - // TODO: Old parser expected error on line 1: unterminated regex literal - DiagnosticSpec(message: "expected expression"), - DiagnosticSpec(message: "extraneous code '/^)' at top level"), + DiagnosticSpec(message: "expected '/' to end regex literal") ] ) } @@ -287,8 +273,7 @@ final class ForwardSlashRegexTests: XCTestCase { } """, diagnostics: [ - DiagnosticSpec(message: "expected expression in 'do' statement"), - // TODO: Old parser expected error on line 3: expected expression after operator + DiagnosticSpec(message: "expected expression after operator") ] ) } @@ -302,9 +287,8 @@ final class ForwardSlashRegexTests: XCTestCase { } """, diagnostics: [ - DiagnosticSpec(message: "expected expression in 'do' statement"), + DiagnosticSpec(message: "expected expression after operator"), DiagnosticSpec(message: "unexpected code '? 0 : 1' in 'do' statement"), - // TODO: Old parser expected error on line 3: expected expression after operator ] ) } @@ -327,9 +311,8 @@ final class ForwardSlashRegexTests: XCTestCase { """, diagnostics: [ // TODO: Old parser expected error on line 3: unary operator cannot be separated from its operand - DiagnosticSpec(message: "expected expression in 'do' statement"), + DiagnosticSpec(message: "expected expression after operator"), DiagnosticSpec(message: "unexpected code '?? /x /' in 'do' statement"), - // TODO: Old parser expected error on line 4: expected expression after operator ] ) } @@ -379,7 +362,7 @@ final class ForwardSlashRegexTests: XCTestCase { diagnostics: [ // TODO: Old parser expected error on line 2: operator with postfix spacing cannot start a subexpression DiagnosticSpec(locationMarker: "1️⃣", message: "consecutive statements on a line must be separated by ';'"), - DiagnosticSpec(locationMarker: "2️⃣", message: "expected expression in prefix operator expression"), + DiagnosticSpec(locationMarker: "2️⃣", message: "expected expression in operator"), DiagnosticSpec(locationMarker: "2️⃣", message: "unexpected code '...' in 'do' statement"), ] ) @@ -393,8 +376,7 @@ final class ForwardSlashRegexTests: XCTestCase { } """, diagnostics: [ - // TODO: Old parser expected error on line 2: expected expression after operator - DiagnosticSpec(message: "expected expression in 'do' statement"), + DiagnosticSpec(message: "expected expression after operator") ] ) } @@ -437,7 +419,7 @@ final class ForwardSlashRegexTests: XCTestCase { foo(/abc/, y: /abc /1️⃣) """, diagnostics: [ - DiagnosticSpec(message: "expected expression in function call"), + DiagnosticSpec(message: "expected expression after operator") ] ) } @@ -471,8 +453,7 @@ final class ForwardSlashRegexTests: XCTestCase { } """, diagnostics: [ - DiagnosticSpec(message: "expected expression in subscript"), - // TODO: Old parser expected error on line 3: expected expression after operator + DiagnosticSpec(message: "expected expression after operator") ] ) } @@ -488,8 +469,7 @@ final class ForwardSlashRegexTests: XCTestCase { } """, diagnostics: [ - DiagnosticSpec(message: "expected expression in 'return' statement"), - // TODO: Old parser expected error on line 6: expected expression after operator + DiagnosticSpec(message: "expected expression after operator") ] ) } @@ -512,8 +492,7 @@ final class ForwardSlashRegexTests: XCTestCase { } """, diagnostics: [ - DiagnosticSpec(message: "expected expression in array element"), - // TODO: Old parser expected error on line 2: expected expression after operator + DiagnosticSpec(message: "expected expression after operator") ] ) } @@ -526,9 +505,8 @@ final class ForwardSlashRegexTests: XCTestCase { } """, diagnostics: [ - DiagnosticSpec(locationMarker: "1️⃣", message: "expected expression in dictionary element"), - // TODO: Old parser expected error on line 2: expected expression after operator - DiagnosticSpec(locationMarker: "2️⃣", message: "expected expression in dictionary element"), + DiagnosticSpec(locationMarker: "1️⃣", message: "expected expression after operator"), + DiagnosticSpec(locationMarker: "2️⃣", message: "expected expression after operator"), ] ) } @@ -589,8 +567,7 @@ final class ForwardSlashRegexTests: XCTestCase { } """, diagnostics: [ - DiagnosticSpec(message: "expected expression in tuple"), - // TODO: Old parser expected error on line 2: expected expression after operator + DiagnosticSpec(message: "expected expression after operator") ] ) } @@ -724,11 +701,10 @@ final class ForwardSlashRegexTests: XCTestCase { // the user hasn't written the member name yet. assertParse( """ - _ = 01️⃣. / 1 / 2 + _ = 0.1️⃣ / 1 / 2 """, diagnostics: [ - // TODO: Old parser expected error on line 1: expected member name following '.' - DiagnosticSpec(message: "extraneous code '. / 1 / 2' at top level"), + DiagnosticSpec(message: "expected name in member access") ] ) } @@ -739,8 +715,7 @@ final class ForwardSlashRegexTests: XCTestCase { _ = 0 . 1️⃣/ 1 / 2 """, diagnostics: [ - // TODO: Old parser expected error on line 1: expected member name following '.' - DiagnosticSpec(message: "expected name in member access"), + DiagnosticSpec(message: "expected name in member access") ] ) } @@ -843,7 +818,7 @@ final class ForwardSlashRegexTests: XCTestCase { } """, diagnostics: [ - DiagnosticSpec(message: "expected pattern in variable"), + DiagnosticSpec(message: "expected pattern in variable") ] ) } @@ -856,8 +831,7 @@ final class ForwardSlashRegexTests: XCTestCase { } """, diagnostics: [ - DiagnosticSpec(message: "expected expression in 'do' statement"), - // TODO: Old parser expected error on line 3: expected expression after operator + DiagnosticSpec(message: "expected expression after operator") ] ) } @@ -870,8 +844,7 @@ final class ForwardSlashRegexTests: XCTestCase { } """, diagnostics: [ - DiagnosticSpec(message: "expected expression in 'do' statement"), - // TODO: Old parser expected error on line 3: expected expression after operator + DiagnosticSpec(message: "expected expression after operator") ] ) } @@ -884,8 +857,7 @@ final class ForwardSlashRegexTests: XCTestCase { } """, diagnostics: [ - DiagnosticSpec(message: "expected expression in 'do' statement"), - // TODO: Old parser expected error on line 3: expected expression after operator + DiagnosticSpec(message: "expected expression after operator") ] ) } @@ -998,7 +970,7 @@ final class ForwardSlashRegexTests: XCTestCase { } """#, diagnostics: [ - DiagnosticSpec(message: "expected ')' to end function call"), + DiagnosticSpec(message: "expected ')' to end function call") ] ) } @@ -1152,12 +1124,10 @@ final class ForwardSlashRegexTests: XCTestCase { func testForwardSlashRegex139() { assertParse( """ - _ = 1️⃣/ + _ = /1️⃣ """, diagnostics: [ - // TODO: Old parser expected error on line 1: unterminated regex literal - DiagnosticSpec(message: "expected expression"), - DiagnosticSpec(message: "extraneous code '/' at top level"), + DiagnosticSpec(message: "expected '/' to end regex literal") ] ) } @@ -1165,12 +1135,10 @@ final class ForwardSlashRegexTests: XCTestCase { func testForwardSlashRegex140() { assertParse( """ - _ = 1️⃣/) + _ = /)1️⃣ """, diagnostics: [ - // TODO: Old parser expected error on line 1: unterminated regex literal - DiagnosticSpec(message: "expected expression"), - DiagnosticSpec(message: "extraneous code '/)' at top level"), + DiagnosticSpec(message: "expected '/' to end regex literal") ] ) } @@ -1189,8 +1157,7 @@ final class ForwardSlashRegexTests: XCTestCase { _ = /\()1️⃣/ """#, diagnostics: [ - // TODO: Old parser expected error on line 1: invalid component of Swift key path - DiagnosticSpec(message: "extraneous code '/' at top level"), + DiagnosticSpec(message: "extraneous code '/' at top level") ] ) } @@ -1204,7 +1171,7 @@ final class ForwardSlashRegexTests: XCTestCase { """#, diagnostics: [ // TODO: Old parser expected note on line 2: to match this opening '(' - DiagnosticSpec(message: "expected ')' to end tuple"), + DiagnosticSpec(message: "expected ')' to end tuple") ] ) } @@ -1217,9 +1184,7 @@ final class ForwardSlashRegexTests: XCTestCase { } """, diagnostics: [ - // TODO: Old parser expected error on line 2: consecutive statements on a line must be separated by ';' - // TODO: Old parser expected error on line 2: expected expression - DiagnosticSpec(message: "unexpected code ')/' in 'do' statement"), + DiagnosticSpec(message: "unexpected code ')/' in 'do' statement") ] ) } @@ -1232,9 +1197,7 @@ final class ForwardSlashRegexTests: XCTestCase { } """, diagnostics: [ - // TODO: Old parser expected error on line 2: consecutive statements on a line must be separated by ';' - // TODO: Old parser expected error on line 2: expected expression - DiagnosticSpec(message: "unexpected code ')/' in 'do' statement"), + DiagnosticSpec(message: "unexpected code ')/' in 'do' statement") ] ) } @@ -1247,7 +1210,6 @@ final class ForwardSlashRegexTests: XCTestCase { } """#, diagnostics: [ - // TODO: Old parser expected error on line 2: expected expression path in Swift key path DiagnosticSpec(locationMarker: "1️⃣", message: "expected root in key path"), DiagnosticSpec(locationMarker: "2️⃣", message: "unexpected code '])/' in 'do' statement"), ] @@ -1258,14 +1220,16 @@ final class ForwardSlashRegexTests: XCTestCase { assertParse( """ _ = ^/x/ - """) + """ + ) } func testForwardSlashRegex148() { assertParse( """ _ = (^/x)/ - """) + """ + ) } func testForwardSlashRegex149() { @@ -1279,10 +1243,11 @@ final class ForwardSlashRegexTests: XCTestCase { func testForwardSlashRegex150() { assertParse( #""" - _ = ^/"/" + _ = ^/"/1️⃣"2️⃣ """#, diagnostics: [ - // TODO: Old parser expected error on line 1: unterminated string literal + DiagnosticSpec(locationMarker: "1️⃣", message: "consecutive statements on a line must be separated by ';'"), + DiagnosticSpec(locationMarker: "2️⃣", message: #"expected '"' to end string literal"#), ] ) } @@ -1290,10 +1255,11 @@ final class ForwardSlashRegexTests: XCTestCase { func testForwardSlashRegex151() { assertParse( #""" - _ = ^/"[/" + _ = ^/"[/1️⃣"2️⃣ """#, diagnostics: [ - // TODO: Old parser expected error on line 1: unterminated string literal + DiagnosticSpec(locationMarker: "1️⃣", message: "consecutive statements on a line must be separated by ';'"), + DiagnosticSpec(locationMarker: "2️⃣", message: #"expected '"' to end string literal"#), ] ) } @@ -1315,6 +1281,7 @@ final class ForwardSlashRegexTests: XCTestCase { } func testForwardSlashRegex157() { + // Okay, as the space is escaped. assertParse( #""" _ = /\ / @@ -1325,12 +1292,10 @@ final class ForwardSlashRegexTests: XCTestCase { func testForwardSlashRegex158() { assertParse( """ - _ = 1️⃣/ / + _ = /1️⃣ / """, diagnostics: [ - // TODO: Old parser expected error on line 1: regex literal may not start with space; add backslash to escape, Fix-It replacements: 6 - 6 = '\' - DiagnosticSpec(message: "expected expression"), - DiagnosticSpec(message: "extraneous code '/ /' at top level"), + DiagnosticSpec(message: "bare slash regex literal may not end with space") ] ) } @@ -1338,13 +1303,10 @@ final class ForwardSlashRegexTests: XCTestCase { func testForwardSlashRegex159() { assertParse( """ - _ = 1️⃣/ / + _ = / 1️⃣ / """, diagnostics: [ - // TODO: Old parser expected error on line 1: regex literal may not start with space; add backslash to escape, Fix-It replacements: 6 - 6 = '\' - // TODO: Old parser expected error on line 1: regex literal may not end with space; use extended literal instead, Fix-It replacements: 5 - 5 = '#', 9 - 9 = '#' - DiagnosticSpec(message: "expected expression"), - DiagnosticSpec(message: "extraneous code '/ /' at top level"), + DiagnosticSpec(message: "bare slash regex literal may not end with space") ] ) } @@ -1376,6 +1338,7 @@ final class ForwardSlashRegexTests: XCTestCase { func testForwardSlashRegex163() { assertParse( """ + """ ) } @@ -1384,13 +1347,11 @@ final class ForwardSlashRegexTests: XCTestCase { // There are intentionally trailing spaces here assertParse( """ - _ = 1️⃣/ + _ = /1️⃣ 2️⃣ """, diagnostics: [ - // TODO: Old parser expected error on line 1: unterminated regex literal - // TODO: Old parser expected error on line 1: regex literal may not start with space; add backslash to escape, Fix-It replacements: 6 - 6 = '\' - DiagnosticSpec(message: "expected expression"), - DiagnosticSpec(message: "extraneous code '/' at top level"), + DiagnosticSpec(locationMarker: "1️⃣", message: "bare slash regex literal may not start with space"), + DiagnosticSpec(locationMarker: "2️⃣", message: "expected '/' to end regex literal"), ] ) } @@ -1399,12 +1360,10 @@ final class ForwardSlashRegexTests: XCTestCase { // There are intentionally trailing spaces here assertParse( """ - _ = 1️⃣/^ + _ = /^ 1️⃣ """, diagnostics: [ - // TODO: Old parser expected error on line 1: unterminated regex literal - DiagnosticSpec(message: "expected expression"), - DiagnosticSpec(message: "extraneous code '/^' at top level"), + DiagnosticSpec(message: "expected '/' to end regex literal") ] ) } @@ -1420,12 +1379,8 @@ final class ForwardSlashRegexTests: XCTestCase { func testForwardSlashRegex168() { assertParse( """ - _ = 1️⃣/)/ - """, - diagnostics: [ - DiagnosticSpec(message: "expected expression"), - DiagnosticSpec(message: "extraneous code '/)/' at top level"), - ] + _ = /)/ + """ ) } @@ -1513,12 +1468,7 @@ final class ForwardSlashRegexTests: XCTestCase { assertParse( """ _ = ^^/1️⃣0xG/ - """, - diagnostics: [ - // FIXME: This should not error - DiagnosticSpec(message: "expected expression in prefix operator expression"), - DiagnosticSpec(message: "extraneous code '0xG/' at top level"), - ] + """ ) } @@ -1526,12 +1476,7 @@ final class ForwardSlashRegexTests: XCTestCase { assertParse( """ _ = ^^/1️⃣0oG/ - """, - diagnostics: [ - // FIXME: This should not error - DiagnosticSpec(message: "expected expression in prefix operator expression"), - DiagnosticSpec(message: "extraneous code '0oG/' at top level"), - ] + """ ) } @@ -1539,11 +1484,7 @@ final class ForwardSlashRegexTests: XCTestCase { assertParse( #""" _ = ^^/"/1️⃣ - """#, - diagnostics: [ - // FIXME: This should not error - DiagnosticSpec(message: #"expected '"' to end string literal"#), - ] + """# ) } @@ -1551,11 +1492,7 @@ final class ForwardSlashRegexTests: XCTestCase { assertParse( """ _ = ^^/'/1️⃣ - """, - diagnostics: [ - // FIXME: This should not error - DiagnosticSpec(message: "expected ''' in string literal"), - ] + """ ) } @@ -1571,12 +1508,7 @@ final class ForwardSlashRegexTests: XCTestCase { assertParse( """ _ = (^^/1️⃣0xG/) - """, - diagnostics: [ - // FIXME: This should not error - DiagnosticSpec(message: "expected expression in prefix operator expression"), - DiagnosticSpec(message: "unexpected code '0xG/' in tuple"), - ] + """ ) } @@ -1584,12 +1516,7 @@ final class ForwardSlashRegexTests: XCTestCase { assertParse( """ _ = (^^/1️⃣0oG/) - """, - diagnostics: [ - // FIXME: This should not error - DiagnosticSpec(message: "expected expression in prefix operator expression"), - DiagnosticSpec(message: "unexpected code '0oG/' in tuple"), - ] + """ ) } @@ -1597,12 +1524,7 @@ final class ForwardSlashRegexTests: XCTestCase { assertParse( #""" _ = (^^/"/)1️⃣ - """#, - diagnostics: [ - // FIXME: This should not error - DiagnosticSpec(message: #"expected '"' to end string literal"#), - DiagnosticSpec(message: "expected ')' to end tuple"), - ] + """# ) } @@ -1610,12 +1532,7 @@ final class ForwardSlashRegexTests: XCTestCase { assertParse( """ _ = (^^/'/)1️⃣ - """, - diagnostics: [ - // FIXME: This should not error - DiagnosticSpec(message: "expected ''' in string literal"), - DiagnosticSpec(message: "expected ')' to end tuple"), - ] + """ ) } diff --git a/Tests/SwiftParserTest/translated/RegexParseEndOfBufferTests.swift b/Tests/SwiftParserTest/translated/RegexParseEndOfBufferTests.swift index ea500fe71f8..073e660cb5b 100644 --- a/Tests/SwiftParserTest/translated/RegexParseEndOfBufferTests.swift +++ b/Tests/SwiftParserTest/translated/RegexParseEndOfBufferTests.swift @@ -5,12 +5,9 @@ import XCTest final class RegexParseEndOfBufferTests: XCTestCase { func testRegexParseEndOfBuffer1() { assertParse( - """ - // Note there is purposefully no trailing newline here. - var unterminated = 1️⃣#/(xy - """, + "var unterminated = #/(xy1️⃣", diagnostics: [ - DiagnosticSpec(message: "unterminated regex literal") + DiagnosticSpec(message: "expected '/#' to end regex literal") ] ) } diff --git a/Tests/SwiftParserTest/translated/RegexParseErrorTests.swift b/Tests/SwiftParserTest/translated/RegexParseErrorTests.swift index 22d504f1c7b..b172af0ecfe 100644 --- a/Tests/SwiftParserTest/translated/RegexParseErrorTests.swift +++ b/Tests/SwiftParserTest/translated/RegexParseErrorTests.swift @@ -22,13 +22,8 @@ final class RegexParseErrorTests: XCTestCase { func testRegexParseError4() { assertParse( """ - _ = 1️⃣/)/ - """, - diagnostics: [ - DiagnosticSpec(message: "expected expression"), - DiagnosticSpec(message: "extraneous code '/)/' at top level"), - // TODO: Old parser expected error on line 1: closing ')' does not balance any groups openings - ] + _ = /)/ + """ ) } @@ -43,10 +38,10 @@ final class RegexParseErrorTests: XCTestCase { func testRegexParseError6() { assertParse( #""" - _ = 1️⃣#/\\/''/ + _ = #/\\/''/1️⃣ """#, diagnostics: [ - DiagnosticSpec(message: "unterminated regex literal") + DiagnosticSpec(message: "expected '#' to end regex literal") ] ) } @@ -54,10 +49,10 @@ final class RegexParseErrorTests: XCTestCase { func testRegexParseError7() { assertParse( #""" - _ = 1️⃣#/\| + _ = #/\|1️⃣ """#, diagnostics: [ - DiagnosticSpec(message: "unterminated regex literal") + DiagnosticSpec(message: "expected '/#' to end regex literal") ] ) } @@ -65,10 +60,10 @@ final class RegexParseErrorTests: XCTestCase { func testRegexParseError8() { assertParse( """ - _ = 1️⃣#// + _ = #//1️⃣ """, diagnostics: [ - DiagnosticSpec(message: "unterminated regex literal") + DiagnosticSpec(message: "expected '#' to end regex literal") ] ) } @@ -76,10 +71,10 @@ final class RegexParseErrorTests: XCTestCase { func testRegexParseError9() { assertParse( """ - _ = 1️⃣#/xy + _ = #/xy1️⃣ """, diagnostics: [ - DiagnosticSpec(message: "unterminated regex literal") + DiagnosticSpec(message: "expected '/#' to end regex literal") ] ) } @@ -120,11 +115,11 @@ final class RegexParseErrorTests: XCTestCase { assertParse( """ do { - _ = 1️⃣#/(?'a + _ = #/(?'a1️⃣ } """, diagnostics: [ - DiagnosticSpec(message: "unterminated regex literal") + DiagnosticSpec(message: "expected '/#' to end regex literal") ] ) } @@ -147,8 +142,7 @@ final class RegexParseErrorTests: XCTestCase { """#, diagnostics: [ DiagnosticSpec(locationMarker: "1️⃣", message: "expected root in key path"), - // TODO: Old parser expected error on line 3: expected expression path in Swift key path - DiagnosticSpec(locationMarker: "2️⃣", message: "expected expression in 'do' statement"), + DiagnosticSpec(locationMarker: "2️⃣", message: "expected expression after operator"), ] ) } @@ -157,13 +151,13 @@ final class RegexParseErrorTests: XCTestCase { assertParse( #""" do { - _ = 1️⃣#/\ + _ = #/\1️⃣ /#2️⃣ } """#, diagnostics: [ - DiagnosticSpec(locationMarker: "1️⃣", message: "unterminated regex literal"), - DiagnosticSpec(locationMarker: "2️⃣", message: "expected identifier in pound literal expression") + DiagnosticSpec(locationMarker: "1️⃣", message: "expected '/#' to end regex literal"), + DiagnosticSpec(locationMarker: "2️⃣", message: "expected identifier in macro expansion"), ] ) } From b9e576ae70322b8c19312666025d14e50712ccee Mon Sep 17 00:00:00 2001 From: Hamish Knight Date: Wed, 29 Mar 2023 00:25:22 +0100 Subject: [PATCH 08/12] Update generated code --- .../generated/SyntaxClassification.swift | 10 +- .../generated/TokenSpecStaticMembers.swift | 12 +- Sources/SwiftSyntax/generated/TokenKind.swift | 67 +++++-- Sources/SwiftSyntax/generated/Tokens.swift | 34 +++- .../generated/raw/RawSyntaxNodes.swift | 70 ++++++-- .../generated/raw/RawSyntaxValidation.swift | 12 +- .../syntaxNodes/SyntaxExprNodes.swift | 169 ++++++++++++++++-- 7 files changed, 328 insertions(+), 46 deletions(-) diff --git a/Sources/IDEUtils/generated/SyntaxClassification.swift b/Sources/IDEUtils/generated/SyntaxClassification.swift index 697d5344a52..b4a9f336ec7 100644 --- a/Sources/IDEUtils/generated/SyntaxClassification.swift +++ b/Sources/IDEUtils/generated/SyntaxClassification.swift @@ -47,6 +47,8 @@ public enum SyntaxClassification { case operatorIdentifier /// A `#` keyword like `#warning`. case poundDirectiveKeyword + /// A regex literal, including multiline regex literals. + case regexLiteral /// The opening and closing parenthesis of string interpolation. case stringInterpolationAnchor /// A string literal including multiline string literals. @@ -138,6 +140,8 @@ extension RawTokenKind { return .none case .exclamationMark: return .none + case .extendedRegexDelimiter: + return .regexLiteral case .floatingLiteral: return .floatingLiteral case .identifier: @@ -186,8 +190,10 @@ extension RawTokenKind { return .operatorIdentifier case .rawStringDelimiter: return .none - case .regexLiteral: - return .none + case .regexLiteralPattern: + return .regexLiteral + case .regexSlash: + return .regexLiteral case .rightAngle: return .none case .rightBrace: diff --git a/Sources/SwiftParser/generated/TokenSpecStaticMembers.swift b/Sources/SwiftParser/generated/TokenSpecStaticMembers.swift index 438d0d4b8e5..dd90a837195 100644 --- a/Sources/SwiftParser/generated/TokenSpecStaticMembers.swift +++ b/Sources/SwiftParser/generated/TokenSpecStaticMembers.swift @@ -63,6 +63,10 @@ extension TokenSpec { return TokenSpec(.exclamationMark) } + static var extendedRegexDelimiter: TokenSpec { + return TokenSpec(.extendedRegexDelimiter) + } + static var floatingLiteral: TokenSpec { return TokenSpec(.floatingLiteral) } @@ -155,8 +159,12 @@ extension TokenSpec { return TokenSpec(.rawStringDelimiter) } - static var regexLiteral: TokenSpec { - return TokenSpec(.regexLiteral) + static var regexLiteralPattern: TokenSpec { + return TokenSpec(.regexLiteralPattern) + } + + static var regexSlash: TokenSpec { + return TokenSpec(.regexSlash) } static var rightAngle: TokenSpec { diff --git a/Sources/SwiftSyntax/generated/TokenKind.swift b/Sources/SwiftSyntax/generated/TokenKind.swift index 739bd58864b..227f514f270 100644 --- a/Sources/SwiftSyntax/generated/TokenKind.swift +++ b/Sources/SwiftSyntax/generated/TokenKind.swift @@ -27,6 +27,7 @@ public enum TokenKind: Hashable { case ellipsis case equal case exclamationMark + case extendedRegexDelimiter(String) case floatingLiteral(String) case identifier(String) case infixQuestionMark @@ -51,7 +52,8 @@ public enum TokenKind: Hashable { case prefixAmpersand case prefixOperator(String) case rawStringDelimiter(String) - case regexLiteral(String) + case regexLiteralPattern(String) + case regexSlash case rightAngle case rightBrace case rightParen @@ -89,6 +91,8 @@ public enum TokenKind: Hashable { return #"="# case .exclamationMark: return #"!"# + case .extendedRegexDelimiter(let text): + return text case .floatingLiteral(let text): return text case .identifier(let text): @@ -137,8 +141,10 @@ public enum TokenKind: Hashable { return text case .rawStringDelimiter(let text): return text - case .regexLiteral(let text): + case .regexLiteralPattern(let text): return text + case .regexSlash: + return #"/"# case .rightAngle: return #">"# case .rightBrace: @@ -222,6 +228,8 @@ public enum TokenKind: Hashable { return #"#unavailable"# case .prefixAmpersand: return #"&"# + case .regexSlash: + return #"/"# case .rightAngle: return #">"# case .rightBrace: @@ -271,6 +279,8 @@ public enum TokenKind: Hashable { return #"="# case .exclamationMark: return #"!"# + case .extendedRegexDelimiter: + return #"extended delimiter"# case .floatingLiteral: return #"floating literal"# case .identifier: @@ -317,8 +327,10 @@ public enum TokenKind: Hashable { return #"prefix operator"# case .rawStringDelimiter: return #"raw string delimiter"# - case .regexLiteral: - return #"regex literal"# + case .regexLiteralPattern: + return #"regex pattern"# + case .regexSlash: + return #"/"# case .rightAngle: return #">"# case .rightBrace: @@ -375,6 +387,8 @@ public enum TokenKind: Hashable { return false case .exclamationMark: return false + case .extendedRegexDelimiter: + return false case .floatingLiteral: return false case .identifier: @@ -421,7 +435,9 @@ public enum TokenKind: Hashable { return false case .rawStringDelimiter: return false - case .regexLiteral: + case .regexLiteralPattern: + return false + case .regexSlash: return false case .rightAngle: return false @@ -479,6 +495,8 @@ public enum TokenKind: Hashable { return true case .exclamationMark: return true + case .extendedRegexDelimiter: + return false case .floatingLiteral: return false case .identifier: @@ -527,8 +545,10 @@ public enum TokenKind: Hashable { return false case .rawStringDelimiter: return false - case .regexLiteral: + case .regexLiteralPattern: return false + case .regexSlash: + return true case .rightAngle: return true case .rightBrace: @@ -580,6 +600,8 @@ extension TokenKind: Equatable { return true case (.exclamationMark, .exclamationMark): return true + case (.extendedRegexDelimiter(let lhsText), .extendedRegexDelimiter(let rhsText)): + return lhsText == rhsText case (.floatingLiteral(let lhsText), .floatingLiteral(let rhsText)): return lhsText == rhsText case (.identifier(let lhsText), .identifier(let rhsText)): @@ -628,8 +650,10 @@ extension TokenKind: Equatable { return lhsText == rhsText case (.rawStringDelimiter(let lhsText), .rawStringDelimiter(let rhsText)): return lhsText == rhsText - case (.regexLiteral(let lhsText), .regexLiteral(let rhsText)): + case (.regexLiteralPattern(let lhsText), .regexLiteralPattern(let rhsText)): return lhsText == rhsText + case (.regexSlash, .regexSlash): + return true case (.rightAngle, .rightAngle): return true case (.rightBrace, .rightBrace): @@ -674,6 +698,7 @@ public enum RawTokenKind: UInt8, Equatable, Hashable { case ellipsis case equal case exclamationMark + case extendedRegexDelimiter case floatingLiteral case identifier case infixQuestionMark @@ -698,7 +723,8 @@ public enum RawTokenKind: UInt8, Equatable, Hashable { case prefixAmpersand case prefixOperator case rawStringDelimiter - case regexLiteral + case regexLiteralPattern + case regexSlash case rightAngle case rightBrace case rightParen @@ -767,6 +793,8 @@ public enum RawTokenKind: UInt8, Equatable, Hashable { return #"#unavailable"# case .prefixAmpersand: return #"&"# + case .regexSlash: + return #"/"# case .rightAngle: return #">"# case .rightBrace: @@ -819,6 +847,8 @@ public enum RawTokenKind: UInt8, Equatable, Hashable { return true case .exclamationMark: return true + case .extendedRegexDelimiter: + return false case .floatingLiteral: return false case .identifier: @@ -867,8 +897,10 @@ public enum RawTokenKind: UInt8, Equatable, Hashable { return false case .rawStringDelimiter: return false - case .regexLiteral: + case .regexLiteralPattern: return false + case .regexSlash: + return true case .rightAngle: return true case .rightBrace: @@ -931,6 +963,8 @@ extension TokenKind { case .exclamationMark: precondition(text.isEmpty || rawKind.defaultText.map(String.init) == text) return .exclamationMark + case .extendedRegexDelimiter: + return .extendedRegexDelimiter(text) case .floatingLiteral: return .floatingLiteral(text) case .identifier: @@ -999,8 +1033,11 @@ extension TokenKind { return .prefixOperator(text) case .rawStringDelimiter: return .rawStringDelimiter(text) - case .regexLiteral: - return .regexLiteral(text) + case .regexLiteralPattern: + return .regexLiteralPattern(text) + case .regexSlash: + precondition(text.isEmpty || rawKind.defaultText.map(String.init) == text) + return .regexSlash case .rightAngle: precondition(text.isEmpty || rawKind.defaultText.map(String.init) == text) return .rightAngle @@ -1061,6 +1098,8 @@ extension TokenKind { return (.equal, nil) case .exclamationMark: return (.exclamationMark, nil) + case .extendedRegexDelimiter(let str): + return (.extendedRegexDelimiter, str) case .floatingLiteral(let str): return (.floatingLiteral, str) case .identifier(let str): @@ -1109,8 +1148,10 @@ extension TokenKind { return (.prefixOperator, str) case .rawStringDelimiter(let str): return (.rawStringDelimiter, str) - case .regexLiteral(let str): - return (.regexLiteral, str) + case .regexLiteralPattern(let str): + return (.regexLiteralPattern, str) + case .regexSlash: + return (.regexSlash, nil) case .rightAngle: return (.rightAngle, nil) case .rightBrace: diff --git a/Sources/SwiftSyntax/generated/Tokens.swift b/Sources/SwiftSyntax/generated/Tokens.swift index 5ff628cec23..3ad25b165e9 100644 --- a/Sources/SwiftSyntax/generated/Tokens.swift +++ b/Sources/SwiftSyntax/generated/Tokens.swift @@ -171,6 +171,22 @@ extension TokenSyntax { ) } + public static func extendedRegexDelimiter( + _ text: String, + leadingTrivia: Trivia = [], + trailingTrivia: Trivia = [], + presence: SourcePresence = .present + + ) -> TokenSyntax { + return TokenSyntax( + .extendedRegexDelimiter(text), + leadingTrivia: leadingTrivia, + trailingTrivia: trailingTrivia, + presence: presence + + ) + } + public static func floatingLiteral( _ text: String, leadingTrivia: Trivia = [], @@ -521,7 +537,7 @@ extension TokenSyntax { ) } - public static func regexLiteral( + public static func regexLiteralPattern( _ text: String, leadingTrivia: Trivia = [], trailingTrivia: Trivia = [], @@ -529,7 +545,21 @@ extension TokenSyntax { ) -> TokenSyntax { return TokenSyntax( - .regexLiteral(text), + .regexLiteralPattern(text), + leadingTrivia: leadingTrivia, + trailingTrivia: trailingTrivia, + presence: presence + + ) + } + + public static func regexSlashToken( + leadingTrivia: Trivia = [], + trailingTrivia: Trivia = [], + presence: SourcePresence = .present + ) -> TokenSyntax { + return TokenSyntax( + .regexSlash, leadingTrivia: leadingTrivia, trailingTrivia: trailingTrivia, presence: presence diff --git a/Sources/SwiftSyntax/generated/raw/RawSyntaxNodes.swift b/Sources/SwiftSyntax/generated/raw/RawSyntaxNodes.swift index 39c96776cae..76fca89a259 100644 --- a/Sources/SwiftSyntax/generated/raw/RawSyntaxNodes.swift +++ b/Sources/SwiftSyntax/generated/raw/RawSyntaxNodes.swift @@ -16805,32 +16805,80 @@ public struct RawRegexLiteralExprSyntax: RawExprSyntaxNodeProtocol { } public init( - _ unexpectedBeforeRegex: RawUnexpectedNodesSyntax? = nil, - regex: RawTokenSyntax, - _ unexpectedAfterRegex: RawUnexpectedNodesSyntax? = nil, + _ unexpectedBeforeOpeningPounds: RawUnexpectedNodesSyntax? = nil, + openingPounds: RawTokenSyntax?, + _ unexpectedBetweenOpeningPoundsAndOpenSlash: RawUnexpectedNodesSyntax? = nil, + openSlash: RawTokenSyntax, + _ unexpectedBetweenOpenSlashAndRegexPattern: RawUnexpectedNodesSyntax? = nil, + regexPattern: RawTokenSyntax, + _ unexpectedBetweenRegexPatternAndCloseSlash: RawUnexpectedNodesSyntax? = nil, + closeSlash: RawTokenSyntax, + _ unexpectedBetweenCloseSlashAndClosingPounds: RawUnexpectedNodesSyntax? = nil, + closingPounds: RawTokenSyntax?, + _ unexpectedAfterClosingPounds: RawUnexpectedNodesSyntax? = nil, arena: __shared SyntaxArena ) { let raw = RawSyntax.makeLayout( - kind: .regexLiteralExpr, uninitializedCount: 3, arena: arena) { layout in + kind: .regexLiteralExpr, uninitializedCount: 11, arena: arena) { layout in layout.initialize(repeating: nil) - layout[0] = unexpectedBeforeRegex?.raw - layout[1] = regex.raw - layout[2] = unexpectedAfterRegex?.raw + layout[0] = unexpectedBeforeOpeningPounds?.raw + layout[1] = openingPounds?.raw + layout[2] = unexpectedBetweenOpeningPoundsAndOpenSlash?.raw + layout[3] = openSlash.raw + layout[4] = unexpectedBetweenOpenSlashAndRegexPattern?.raw + layout[5] = regexPattern.raw + layout[6] = unexpectedBetweenRegexPatternAndCloseSlash?.raw + layout[7] = closeSlash.raw + layout[8] = unexpectedBetweenCloseSlashAndClosingPounds?.raw + layout[9] = closingPounds?.raw + layout[10] = unexpectedAfterClosingPounds?.raw } self.init(unchecked: raw) } - public var unexpectedBeforeRegex: RawUnexpectedNodesSyntax? { + public var unexpectedBeforeOpeningPounds: RawUnexpectedNodesSyntax? { layoutView.children[0].map(RawUnexpectedNodesSyntax.init(raw:)) } - public var regex: RawTokenSyntax { - layoutView.children[1].map(RawTokenSyntax.init(raw:))! + public var openingPounds: RawTokenSyntax? { + layoutView.children[1].map(RawTokenSyntax.init(raw:)) } - public var unexpectedAfterRegex: RawUnexpectedNodesSyntax? { + public var unexpectedBetweenOpeningPoundsAndOpenSlash: RawUnexpectedNodesSyntax? { layoutView.children[2].map(RawUnexpectedNodesSyntax.init(raw:)) } + + public var openSlash: RawTokenSyntax { + layoutView.children[3].map(RawTokenSyntax.init(raw:))! + } + + public var unexpectedBetweenOpenSlashAndRegexPattern: RawUnexpectedNodesSyntax? { + layoutView.children[4].map(RawUnexpectedNodesSyntax.init(raw:)) + } + + public var regexPattern: RawTokenSyntax { + layoutView.children[5].map(RawTokenSyntax.init(raw:))! + } + + public var unexpectedBetweenRegexPatternAndCloseSlash: RawUnexpectedNodesSyntax? { + layoutView.children[6].map(RawUnexpectedNodesSyntax.init(raw:)) + } + + public var closeSlash: RawTokenSyntax { + layoutView.children[7].map(RawTokenSyntax.init(raw:))! + } + + public var unexpectedBetweenCloseSlashAndClosingPounds: RawUnexpectedNodesSyntax? { + layoutView.children[8].map(RawUnexpectedNodesSyntax.init(raw:)) + } + + public var closingPounds: RawTokenSyntax? { + layoutView.children[9].map(RawTokenSyntax.init(raw:)) + } + + public var unexpectedAfterClosingPounds: RawUnexpectedNodesSyntax? { + layoutView.children[10].map(RawUnexpectedNodesSyntax.init(raw:)) + } } @_spi(RawSyntax) diff --git a/Sources/SwiftSyntax/generated/raw/RawSyntaxValidation.swift b/Sources/SwiftSyntax/generated/raw/RawSyntaxValidation.swift index c6acf9c5120..5c0b317c1a8 100644 --- a/Sources/SwiftSyntax/generated/raw/RawSyntaxValidation.swift +++ b/Sources/SwiftSyntax/generated/raw/RawSyntaxValidation.swift @@ -2075,10 +2075,18 @@ func validateLayout(layout: RawSyntaxBuffer, as kind: SyntaxKind) { assertNoError(kind, 7, verify(layout[7], as: RawDeclNameArgumentsSyntax?.self)) assertNoError(kind, 8, verify(layout[8], as: RawUnexpectedNodesSyntax?.self)) case .regexLiteralExpr: - assert(layout.count == 3) + assert(layout.count == 11) assertNoError(kind, 0, verify(layout[0], as: RawUnexpectedNodesSyntax?.self)) - assertNoError(kind, 1, verify(layout[1], as: RawTokenSyntax.self, tokenChoices: [.tokenKind(.regexLiteral)])) + assertNoError(kind, 1, verify(layout[1], as: RawTokenSyntax?.self, tokenChoices: [.tokenKind(.extendedRegexDelimiter)])) assertNoError(kind, 2, verify(layout[2], as: RawUnexpectedNodesSyntax?.self)) + assertNoError(kind, 3, verify(layout[3], as: RawTokenSyntax.self, tokenChoices: [.tokenKind(.regexSlash)])) + assertNoError(kind, 4, verify(layout[4], as: RawUnexpectedNodesSyntax?.self)) + assertNoError(kind, 5, verify(layout[5], as: RawTokenSyntax.self, tokenChoices: [.tokenKind(.regexLiteralPattern)])) + assertNoError(kind, 6, verify(layout[6], as: RawUnexpectedNodesSyntax?.self)) + assertNoError(kind, 7, verify(layout[7], as: RawTokenSyntax.self, tokenChoices: [.tokenKind(.regexSlash)])) + assertNoError(kind, 8, verify(layout[8], as: RawUnexpectedNodesSyntax?.self)) + assertNoError(kind, 9, verify(layout[9], as: RawTokenSyntax?.self, tokenChoices: [.tokenKind(.extendedRegexDelimiter)])) + assertNoError(kind, 10, verify(layout[10], as: RawUnexpectedNodesSyntax?.self)) case .repeatWhileStmt: assert(layout.count == 9) assertNoError(kind, 0, verify(layout[0], as: RawUnexpectedNodesSyntax?.self)) diff --git a/Sources/SwiftSyntax/generated/syntaxNodes/SyntaxExprNodes.swift b/Sources/SwiftSyntax/generated/syntaxNodes/SyntaxExprNodes.swift index 780857eea65..ba423b3bf91 100644 --- a/Sources/SwiftSyntax/generated/syntaxNodes/SyntaxExprNodes.swift +++ b/Sources/SwiftSyntax/generated/syntaxNodes/SyntaxExprNodes.swift @@ -5465,16 +5465,48 @@ public struct RegexLiteralExprSyntax: ExprSyntaxProtocol, SyntaxHashable { public init( leadingTrivia: Trivia? = nil, - _ unexpectedBeforeRegex: UnexpectedNodesSyntax? = nil, - regex: TokenSyntax = .regexLiteral("RegexLiteralToken"), - _ unexpectedAfterRegex: UnexpectedNodesSyntax? = nil, + _ unexpectedBeforeOpeningPounds: UnexpectedNodesSyntax? = nil, + openingPounds: TokenSyntax? = nil, + _ unexpectedBetweenOpeningPoundsAndOpenSlash: UnexpectedNodesSyntax? = nil, + openSlash: TokenSyntax = .regexSlashToken(), + _ unexpectedBetweenOpenSlashAndRegexPattern: UnexpectedNodesSyntax? = nil, + regexPattern: TokenSyntax = .regexLiteralPattern("RegexLiteralPatternToken"), + _ unexpectedBetweenRegexPatternAndCloseSlash: UnexpectedNodesSyntax? = nil, + closeSlash: TokenSyntax = .regexSlashToken(), + _ unexpectedBetweenCloseSlashAndClosingPounds: UnexpectedNodesSyntax? = nil, + closingPounds: TokenSyntax? = nil, + _ unexpectedAfterClosingPounds: UnexpectedNodesSyntax? = nil, trailingTrivia: Trivia? = nil ) { // Extend the lifetime of all parameters so their arenas don't get destroyed // before they can be added as children of the new arena. - let data: SyntaxData = withExtendedLifetime((SyntaxArena(), (unexpectedBeforeRegex, regex, unexpectedAfterRegex))) {(arena, _) in - let layout: [RawSyntax?] = [unexpectedBeforeRegex?.raw, regex.raw, unexpectedAfterRegex?.raw] + let data: SyntaxData = withExtendedLifetime((SyntaxArena(), ( + unexpectedBeforeOpeningPounds, + openingPounds, + unexpectedBetweenOpeningPoundsAndOpenSlash, + openSlash, + unexpectedBetweenOpenSlashAndRegexPattern, + regexPattern, + unexpectedBetweenRegexPatternAndCloseSlash, + closeSlash, + unexpectedBetweenCloseSlashAndClosingPounds, + closingPounds, + unexpectedAfterClosingPounds + ))) {(arena, _) in + let layout: [RawSyntax?] = [ + unexpectedBeforeOpeningPounds?.raw, + openingPounds?.raw, + unexpectedBetweenOpeningPoundsAndOpenSlash?.raw, + openSlash.raw, + unexpectedBetweenOpenSlashAndRegexPattern?.raw, + regexPattern.raw, + unexpectedBetweenRegexPatternAndCloseSlash?.raw, + closeSlash.raw, + unexpectedBetweenCloseSlashAndClosingPounds?.raw, + closingPounds?.raw, + unexpectedAfterClosingPounds?.raw + ] let raw = RawSyntax.makeLayout( kind: SyntaxKind.regexLiteralExpr, from: layout, @@ -5487,7 +5519,7 @@ public struct RegexLiteralExprSyntax: ExprSyntaxProtocol, SyntaxHashable { self.init(data) } - public var unexpectedBeforeRegex: UnexpectedNodesSyntax? { + public var unexpectedBeforeOpeningPounds: UnexpectedNodesSyntax? { get { return data.child(at: 0, parent: Syntax(self)).map(UnexpectedNodesSyntax.init) } @@ -5496,16 +5528,16 @@ public struct RegexLiteralExprSyntax: ExprSyntaxProtocol, SyntaxHashable { } } - public var regex: TokenSyntax { + public var openingPounds: TokenSyntax? { get { - return TokenSyntax(data.child(at: 1, parent: Syntax(self))!) + return data.child(at: 1, parent: Syntax(self)).map(TokenSyntax.init) } set(value) { - self = RegexLiteralExprSyntax(data.replacingChild(at: 1, with: value.raw, arena: SyntaxArena())) + self = RegexLiteralExprSyntax(data.replacingChild(at: 1, with: value?.raw, arena: SyntaxArena())) } } - public var unexpectedAfterRegex: UnexpectedNodesSyntax? { + public var unexpectedBetweenOpeningPoundsAndOpenSlash: UnexpectedNodesSyntax? { get { return data.child(at: 2, parent: Syntax(self)).map(UnexpectedNodesSyntax.init) } @@ -5514,8 +5546,92 @@ public struct RegexLiteralExprSyntax: ExprSyntaxProtocol, SyntaxHashable { } } + public var openSlash: TokenSyntax { + get { + return TokenSyntax(data.child(at: 3, parent: Syntax(self))!) + } + set(value) { + self = RegexLiteralExprSyntax(data.replacingChild(at: 3, with: value.raw, arena: SyntaxArena())) + } + } + + public var unexpectedBetweenOpenSlashAndRegexPattern: UnexpectedNodesSyntax? { + get { + return data.child(at: 4, parent: Syntax(self)).map(UnexpectedNodesSyntax.init) + } + set(value) { + self = RegexLiteralExprSyntax(data.replacingChild(at: 4, with: value?.raw, arena: SyntaxArena())) + } + } + + public var regexPattern: TokenSyntax { + get { + return TokenSyntax(data.child(at: 5, parent: Syntax(self))!) + } + set(value) { + self = RegexLiteralExprSyntax(data.replacingChild(at: 5, with: value.raw, arena: SyntaxArena())) + } + } + + public var unexpectedBetweenRegexPatternAndCloseSlash: UnexpectedNodesSyntax? { + get { + return data.child(at: 6, parent: Syntax(self)).map(UnexpectedNodesSyntax.init) + } + set(value) { + self = RegexLiteralExprSyntax(data.replacingChild(at: 6, with: value?.raw, arena: SyntaxArena())) + } + } + + public var closeSlash: TokenSyntax { + get { + return TokenSyntax(data.child(at: 7, parent: Syntax(self))!) + } + set(value) { + self = RegexLiteralExprSyntax(data.replacingChild(at: 7, with: value.raw, arena: SyntaxArena())) + } + } + + public var unexpectedBetweenCloseSlashAndClosingPounds: UnexpectedNodesSyntax? { + get { + return data.child(at: 8, parent: Syntax(self)).map(UnexpectedNodesSyntax.init) + } + set(value) { + self = RegexLiteralExprSyntax(data.replacingChild(at: 8, with: value?.raw, arena: SyntaxArena())) + } + } + + public var closingPounds: TokenSyntax? { + get { + return data.child(at: 9, parent: Syntax(self)).map(TokenSyntax.init) + } + set(value) { + self = RegexLiteralExprSyntax(data.replacingChild(at: 9, with: value?.raw, arena: SyntaxArena())) + } + } + + public var unexpectedAfterClosingPounds: UnexpectedNodesSyntax? { + get { + return data.child(at: 10, parent: Syntax(self)).map(UnexpectedNodesSyntax.init) + } + set(value) { + self = RegexLiteralExprSyntax(data.replacingChild(at: 10, with: value?.raw, arena: SyntaxArena())) + } + } + public static var structure: SyntaxNodeStructure { - return .layout([\Self.unexpectedBeforeRegex, \Self.regex, \Self.unexpectedAfterRegex]) + return .layout([ + \Self.unexpectedBeforeOpeningPounds, + \Self.openingPounds, + \Self.unexpectedBetweenOpeningPoundsAndOpenSlash, + \Self.openSlash, + \Self.unexpectedBetweenOpenSlashAndRegexPattern, + \Self.regexPattern, + \Self.unexpectedBetweenRegexPatternAndCloseSlash, + \Self.closeSlash, + \Self.unexpectedBetweenCloseSlashAndClosingPounds, + \Self.closingPounds, + \Self.unexpectedAfterClosingPounds + ]) } public func childNameForDiagnostics(_ index: SyntaxChildrenIndex) -> String? { @@ -5526,6 +5642,22 @@ public struct RegexLiteralExprSyntax: ExprSyntaxProtocol, SyntaxHashable { return nil case 2: return nil + case 3: + return nil + case 4: + return nil + case 5: + return nil + case 6: + return nil + case 7: + return nil + case 8: + return nil + case 9: + return nil + case 10: + return nil default: fatalError("Invalid index") } @@ -5535,9 +5667,18 @@ public struct RegexLiteralExprSyntax: ExprSyntaxProtocol, SyntaxHashable { extension RegexLiteralExprSyntax: CustomReflectable { public var customMirror: Mirror { return Mirror(self, children: [ - "unexpectedBeforeRegex": unexpectedBeforeRegex.map(Syntax.init)?.asProtocol(SyntaxProtocol.self) as Any , - "regex": Syntax(regex).asProtocol(SyntaxProtocol.self), - "unexpectedAfterRegex": unexpectedAfterRegex.map(Syntax.init)?.asProtocol(SyntaxProtocol.self) as Any ]) + "unexpectedBeforeOpeningPounds": unexpectedBeforeOpeningPounds.map(Syntax.init)?.asProtocol(SyntaxProtocol.self) as Any , + "openingPounds": openingPounds.map(Syntax.init)?.asProtocol(SyntaxProtocol.self) as Any , + "unexpectedBetweenOpeningPoundsAndOpenSlash": unexpectedBetweenOpeningPoundsAndOpenSlash.map(Syntax.init)?.asProtocol(SyntaxProtocol.self) as Any , + "openSlash": Syntax(openSlash).asProtocol(SyntaxProtocol.self), + "unexpectedBetweenOpenSlashAndRegexPattern": unexpectedBetweenOpenSlashAndRegexPattern.map(Syntax.init)?.asProtocol(SyntaxProtocol.self) as Any , + "regexPattern": Syntax(regexPattern).asProtocol(SyntaxProtocol.self), + "unexpectedBetweenRegexPatternAndCloseSlash": unexpectedBetweenRegexPatternAndCloseSlash.map(Syntax.init)?.asProtocol(SyntaxProtocol.self) as Any , + "closeSlash": Syntax(closeSlash).asProtocol(SyntaxProtocol.self), + "unexpectedBetweenCloseSlashAndClosingPounds": unexpectedBetweenCloseSlashAndClosingPounds.map(Syntax.init)?.asProtocol(SyntaxProtocol.self) as Any , + "closingPounds": closingPounds.map(Syntax.init)?.asProtocol(SyntaxProtocol.self) as Any , + "unexpectedAfterClosingPounds": unexpectedAfterClosingPounds.map(Syntax.init)?.asProtocol(SyntaxProtocol.self) as Any + ]) } } From 798c78c65aa3b60229f4e8c5e0a5b3c70d942d4e Mon Sep 17 00:00:00 2001 From: Hamish Knight Date: Thu, 30 Mar 2023 20:06:25 +0100 Subject: [PATCH 09/12] Address review feedback --- Sources/SwiftParser/Expressions.swift | 5 + Sources/SwiftParser/Lexer/Cursor.swift | 85 ++++++-- .../SwiftParser/Lexer/RegexLiteralLexer.swift | 38 +++- Tests/SwiftParserTest/RegexLiteralTests.swift | 110 +++++++++- ...orwardSlashRegexSkippingAllowedTests.swift | 53 ++--- ...orwardSlashRegexSkippingInvalidTests.swift | 34 +-- .../ForwardSlashRegexSkippingTests.swift | 16 +- .../translated/ForwardSlashRegexTests.swift | 202 +++++++++++++----- .../translated/PrefixSlashTests.swift | 48 ++--- .../RegexParseEndOfBufferTests.swift | 13 +- .../translated/RegexParseErrorTests.swift | 22 +- .../translated/RegexTests.swift | 20 +- 12 files changed, 447 insertions(+), 199 deletions(-) diff --git a/Sources/SwiftParser/Expressions.swift b/Sources/SwiftParser/Expressions.swift index a5cade88afa..6346f216ec8 100644 --- a/Sources/SwiftParser/Expressions.swift +++ b/Sources/SwiftParser/Expressions.swift @@ -1442,6 +1442,11 @@ extension Parser { // Parse the opening slash. let (unexpectedBeforeSlash, openSlash) = self.expect(.regexSlash) + // If we had opening pounds, there should be no trivia for the slash. + if let openPounds = openPounds { + precondition(openPounds.trailingTriviaByteLength == 0 && openSlash.leadingTriviaByteLength == 0) + } + // Parse the pattern and closing slash, avoiding recovery or leading trivia // as the lexer should provide the tokens exactly in order without trivia, // otherwise they should be treated as missing. diff --git a/Sources/SwiftParser/Lexer/Cursor.swift b/Sources/SwiftParser/Lexer/Cursor.swift index 4d618b3edd9..bb2e231d732 100644 --- a/Sources/SwiftParser/Lexer/Cursor.swift +++ b/Sources/SwiftParser/Lexer/Cursor.swift @@ -57,9 +57,10 @@ extension Lexer.Cursor { /// A narrow mode that's used for 'try?' and 'try!' to ensure we prefer to /// lex a regex literal rather than a binary operator. This is needed as the - /// last token will be a postfix operator, which would normally indicate a - /// binary operator is expected next, but in this case we know it must be an - /// expression. See the comment in `tryScanOperatorAsRegexLiteral` for more info. + /// `previousTokenKind` will be `.postfixOperator`, which would normally + /// indicate a binary operator is expected next, but in this case we know it + /// must be an expression. See the comment in + /// `tryScanOperatorAsRegexLiteral` for more info. /// NOTE: This is a complete hack, do not add new uses of this. case preferRegexOverBinaryOperator @@ -93,7 +94,8 @@ extension Lexer.Cursor { case inStringInterpolation(stringLiteralKind: StringLiteralKind, parenCount: Int) /// We have encountered a regex literal, and have its tokens to work - /// through. + /// through. `lexemes` is a pointer to the lexemes allocated in the state + /// stack bump pointer allocator. case inRegexLiteral(index: UInt8, lexemes: UnsafePointer) /// The mode in which leading trivia should be lexed for this state or `nil` @@ -212,6 +214,7 @@ extension Lexer.Cursor { self.kind = kind self.position = position } + init(_ kind: TokenDiagnostic.Kind, position: Lexer.Cursor) { self.init(kind, position: position.position) } @@ -239,6 +242,9 @@ extension Lexer { /// If we have already lexed a token, the kind of the previously lexed token var previousTokenKind: RawTokenKind? + + /// If the `previousTokenKind` is `.keyword`, the keyword kind. Otherwise + /// `nil`. var previousKeyword: Keyword? private var stateStack: StateStack = StateStack() @@ -309,18 +315,56 @@ extension Lexer { /// for this lexeme. let trailingTriviaLexingMode: Lexer.Cursor.TriviaLexingMode? - init( + /// If `tokenKind` is `.keyword`, the kind of keyword produced, otherwise + /// `nil`. + let keywordKind: Keyword? + + private init( _ tokenKind: RawTokenKind, - flags: Lexer.Lexeme.Flags = [], - error: Cursor.LexingDiagnostic? = nil, - stateTransition: StateTransition? = nil, - trailingTriviaLexingMode: Lexer.Cursor.TriviaLexingMode? = nil + flags: Lexer.Lexeme.Flags, + error: Cursor.LexingDiagnostic?, + stateTransition: StateTransition?, + trailingTriviaLexingMode: Lexer.Cursor.TriviaLexingMode?, + keywordKind: Keyword? ) { self.tokenKind = tokenKind self.flags = flags self.error = error self.stateTransition = stateTransition self.trailingTriviaLexingMode = trailingTriviaLexingMode + self.keywordKind = keywordKind + } + + /// Create a lexer result. Note that keywords should use `Result.keyword` + /// instead. + init( + _ tokenKind: RawTokenKind, + flags: Lexer.Lexeme.Flags = [], + error: Cursor.LexingDiagnostic? = nil, + stateTransition: StateTransition? = nil, + trailingTriviaLexingMode: Lexer.Cursor.TriviaLexingMode? = nil + ) { + precondition(tokenKind != .keyword, "Use Result.keyword instead") + self.init( + tokenKind, + flags: flags, + error: error, + stateTransition: stateTransition, + trailingTriviaLexingMode: trailingTriviaLexingMode, + keywordKind: nil + ) + } + + /// Produce a lexer result for a given keyword. + static func keyword(_ kind: Keyword) -> Self { + Self( + .keyword, + flags: [], + error: nil, + stateTransition: nil, + trailingTriviaLexingMode: nil, + keywordKind: kind + ) } } } @@ -366,7 +410,7 @@ extension Lexer.Cursor { let result: Lexer.Result switch currentState { case .normal: - result = lexNormal(sourceBufferStart: sourceBufferStart) + result = lexNormal(sourceBufferStart: sourceBufferStart, preferRegexOverBinaryOperator: false) case .preferRegexOverBinaryOperator: // In this state we lex a single token with the flag set, and then pop the state. result = lexNormal(sourceBufferStart: sourceBufferStart, preferRegexOverBinaryOperator: true) @@ -420,7 +464,7 @@ extension Lexer.Cursor { cursor: cursor ) self.previousTokenKind = result.tokenKind - self.previousKeyword = result.tokenKind == .keyword ? Keyword(lexeme.tokenText)! : nil + self.previousKeyword = result.keywordKind return lexeme } @@ -554,6 +598,7 @@ extension Lexer.Cursor.Position { self.input = UnsafeBufferPointer(rebasing: input) return c } + /// Advance the cursor position by `n` bytes. The offset must be valid. func advanced(by n: Int) -> Self { precondition(n > 0) @@ -824,7 +869,7 @@ extension Lexer.Cursor { extension Lexer.Cursor { private mutating func lexNormal( sourceBufferStart: Lexer.Cursor, - preferRegexOverBinaryOperator: Bool = false + preferRegexOverBinaryOperator: Bool ) -> Lexer.Result { switch self.peek() { case UInt8(ascii: "@"): _ = self.advance(); return Lexer.Result(.atSign) @@ -1010,7 +1055,7 @@ extension Lexer.Cursor { return Lexer.Result(.stringSegment, stateTransition: .pop) default: // If we haven't reached the end of the string interpolation, lex as if we were in a normal expression. - return self.lexNormal(sourceBufferStart: sourceBufferStart) + return self.lexNormal(sourceBufferStart: sourceBufferStart, preferRegexOverBinaryOperator: false) } } } @@ -1916,7 +1961,7 @@ extension Lexer.Cursor { let text = tokStart.text(upTo: self) if let keyword = Keyword(text), keyword.isLexerClassified { - return Lexer.Result(.keyword) + return Lexer.Result.keyword(keyword) } else if text == "_" { return Lexer.Result(.wildcard) } else { @@ -1977,7 +2022,7 @@ extension Lexer.Cursor { case UInt8(ascii: "?"): return .postfixQuestionMark default: - fatalError("Must be at '!' or '?'") + preconditionFailure("Must be at '!' or '?'") } }() _ = self.advance() @@ -2006,17 +2051,17 @@ extension Lexer.Cursor { // Check to see if we have a regex literal starting in the operator. do { - var ptr = tokStart - while ptr.input.baseAddress! < self.input.baseAddress! { + var regexScan = tokStart + while regexScan.input.baseAddress! < self.input.baseAddress! { // Scan for the first '/' in the operator to see if it starts a regex // literal. - guard ptr.is(at: "/") else { - _ = ptr.advance() + guard regexScan.is(at: "/") else { + _ = regexScan.advance() continue } guard let result = self.tryLexOperatorAsRegexLiteral( - at: ptr, + at: regexScan, operatorStart: tokStart, operatorEnd: self, sourceBufferStart: sourceBufferStart, diff --git a/Sources/SwiftParser/Lexer/RegexLiteralLexer.swift b/Sources/SwiftParser/Lexer/RegexLiteralLexer.swift index 2e3274a41bd..5d8a67058c7 100644 --- a/Sources/SwiftParser/Lexer/RegexLiteralLexer.swift +++ b/Sources/SwiftParser/Lexer/RegexLiteralLexer.swift @@ -15,9 +15,17 @@ /// A separate lexer specifically for regex literals. fileprivate struct RegexLiteralLexer { enum LexResult { + /// Continue the lex, this is returned from `lexPatternCharacter` when + /// it successfully lexed a character. case `continue` + + /// The lexing has finished successfully. case done + + /// This is not, in fact, a regex. case notARegex + + /// We have an unterminated regex. case unterminated } @@ -29,10 +37,14 @@ fileprivate struct RegexLiteralLexer { private var firstNewline: Lexer.Cursor? private var isMultiline: Bool { firstNewline != nil } - /// Tracks the current group depth, used to enforce the heuristic that a bare - /// slash regex literal with an unbalanced ')' should be treated as an + /// Tracks the current group '(' depth, used to enforce the heuristic that a + /// bare slash regex literal with an unbalanced ')' should be treated as an /// operator instead. private var groupDepth = 0 + + /// Tracks the current '[' custom character class depth, used to ensure we + /// don't count '(' and ')' characters in a custom character class as counting + /// as group characters. private var customCharacterClassDepth = 0 /// Tracks the last unescaped space or tab character, used to enforce that a @@ -58,7 +70,7 @@ fileprivate struct RegexLiteralLexer { } /// Attempt to lex a character of the regex pattern. - mutating func lexPatternCharacter(escaped: Bool = false) -> LexResult { + private mutating func lexPatternCharacter(escaped: Bool) -> LexResult { if cursor.isAtEndOfFile { // We've hit the end of the buffer. In multi-line mode, we don't want to // skip over what is likely otherwise valid Swift code, so resume from the @@ -136,7 +148,7 @@ fileprivate struct RegexLiteralLexer { } /// Attempt to eat a the closing delimiter. - mutating func tryEatEnding() -> LexResult? { + private mutating func tryEatEnding() -> LexResult? { let openPoundCount = builder.numOpenPounds let slashBegin = cursor var newCursor = cursor @@ -209,7 +221,7 @@ fileprivate struct RegexLiteralLexer { return .done } - mutating func lexImpl() -> LexResult { + private mutating func lexImpl() -> LexResult { // We can consume any number of pound signs. var poundCount = 0 while cursor.advance(matching: "#") { @@ -276,7 +288,7 @@ fileprivate struct RegexLiteralLexer { if let result = tryEatEnding() { return result } - switch lexPatternCharacter() { + switch lexPatternCharacter(escaped: false) { case .continue: continue case let result: @@ -288,7 +300,7 @@ fileprivate struct RegexLiteralLexer { mutating func lex() -> RegexLiteralLexemes? { switch lexImpl() { case .continue: - fatalError("Not a valid result") + preconditionFailure("Not a valid result") case .notARegex: return nil case .unterminated where !mustBeRegex: @@ -337,6 +349,7 @@ extension RegexLiteralLexemes.Element { case closingSlash case closingPounds } + /// Retrieve the actual token kind. var tokenKind: RawTokenKind { switch kind { @@ -370,7 +383,8 @@ extension RegexLiteralLexemes { /// A builder type for the regex literal lexer. /// /// NOTE: This is stored for the regex literal lexer state, so should be kept - /// as small as possible. + /// as small as possible. Additionally, it is allocated using a bump pointer + /// allocator, so must remain a POD type (i.e no classes). fileprivate struct Builder { private(set) var numOpenPounds: Int = 0 private(set) var patternByteLength: Int = 0 @@ -415,6 +429,7 @@ extension RegexLiteralLexemes { _patternErrorOffset = start.distance(to: newValue.position) } } + var hasPounds: Bool { numOpenPounds > 0 } } } @@ -487,7 +502,12 @@ extension RegexLiteralLexemes.Builder { at cursor: Lexer.Cursor ) { precondition(lastLexemeKind == .openingSlash) - patternError = .init(kind, position: cursor) + + // Only record if we don't already have a pattern error, we want to prefer + // the first error we encounter. + if patternError == nil { + patternError = .init(kind, position: cursor) + } } /// Finish regex literal lexing. diff --git a/Tests/SwiftParserTest/RegexLiteralTests.swift b/Tests/SwiftParserTest/RegexLiteralTests.swift index e67c67f8e72..21eb8ccb855 100644 --- a/Tests/SwiftParserTest/RegexLiteralTests.swift +++ b/Tests/SwiftParserTest/RegexLiteralTests.swift @@ -1,3 +1,15 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2023 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// + @_spi(RawSyntax) import SwiftSyntax @_spi(RawSyntax) import SwiftParser import XCTest @@ -10,6 +22,7 @@ final class RegexLiteralTests: XCTestCase { """# ) } + func testForwardSlash2() { assertParse( """ @@ -55,6 +68,7 @@ final class RegexLiteralTests: XCTestCase { ] ) } + func testUnterminated2() { assertParse( #""" @@ -65,6 +79,7 @@ final class RegexLiteralTests: XCTestCase { ] ) } + func testUnterminated3() { assertParse( #""" @@ -75,6 +90,7 @@ final class RegexLiteralTests: XCTestCase { ] ) } + func testUnterminated4() { assertParse( #""" @@ -85,6 +101,7 @@ final class RegexLiteralTests: XCTestCase { ] ) } + func testUnterminated5() { assertParse( #""" @@ -95,6 +112,7 @@ final class RegexLiteralTests: XCTestCase { ] ) } + func testUnterminated6() { assertParse( #""" @@ -105,6 +123,7 @@ final class RegexLiteralTests: XCTestCase { ] ) } + func testUnterminated7() { assertParse( #""" @@ -115,6 +134,7 @@ final class RegexLiteralTests: XCTestCase { ] ) } + func testUnterminated8() { assertParse( #""" @@ -125,6 +145,7 @@ final class RegexLiteralTests: XCTestCase { ] ) } + func testUnterminated9() { assertParse( #""" @@ -135,6 +156,7 @@ final class RegexLiteralTests: XCTestCase { ] ) } + func testUnterminated10() { assertParse( #""" @@ -146,6 +168,7 @@ final class RegexLiteralTests: XCTestCase { fixedSource: "##/##/##" ) } + func testUnterminated11() { assertParse( #""" @@ -157,6 +180,7 @@ final class RegexLiteralTests: XCTestCase { fixedSource: "##/###/##" ) } + func testUnterminated12() { assertParse( #""" @@ -168,6 +192,7 @@ final class RegexLiteralTests: XCTestCase { fixedSource: #"#/\/#/#"# ) } + func testUnterminated13() { assertParse( #""" @@ -179,6 +204,7 @@ final class RegexLiteralTests: XCTestCase { fixedSource: "##/abc/#def/##" ) } + func testUnterminated14() { assertParse( #""" @@ -198,6 +224,7 @@ final class RegexLiteralTests: XCTestCase { """# ) } + func testTerminated2() { assertParse( #""" @@ -205,6 +232,7 @@ final class RegexLiteralTests: XCTestCase { """# ) } + func testTerminated3() { assertParse( #""" @@ -212,6 +240,7 @@ final class RegexLiteralTests: XCTestCase { """# ) } + func testTerminated4() { assertParse( #""" @@ -219,6 +248,7 @@ final class RegexLiteralTests: XCTestCase { """# ) } + func testTerminated5() { assertParse( #""" @@ -226,6 +256,7 @@ final class RegexLiteralTests: XCTestCase { """# ) } + func testTerminated6() { assertParse( #""" @@ -233,6 +264,7 @@ final class RegexLiteralTests: XCTestCase { """# ) } + func testTerminated7() { assertParse( #""" @@ -251,6 +283,7 @@ final class RegexLiteralTests: XCTestCase { ] ) } + func testUnprintable2() { assertParse( """ @@ -273,6 +306,7 @@ final class RegexLiteralTests: XCTestCase { ] ) } + func testMultiline2() { assertParse( """ @@ -285,6 +319,7 @@ final class RegexLiteralTests: XCTestCase { ] ) } + func testMultiline3() { assertParse( """ @@ -297,6 +332,7 @@ final class RegexLiteralTests: XCTestCase { ] ) } + func testMultiline4() { assertParse( """ @@ -306,6 +342,7 @@ final class RegexLiteralTests: XCTestCase { """ ) } + func testMultiline5() { assertParse( """ @@ -317,6 +354,7 @@ final class RegexLiteralTests: XCTestCase { ] ) } + func testOpeningSpace1() { assertParse( """ @@ -327,6 +365,7 @@ final class RegexLiteralTests: XCTestCase { ] ) } + func testOpeningSpace2() { assertParse( """ @@ -337,6 +376,7 @@ final class RegexLiteralTests: XCTestCase { ] ) } + func testOpeningSpace3() { assertParse( """ @@ -344,6 +384,7 @@ final class RegexLiteralTests: XCTestCase { """ ) } + func testClosingSpace1() { assertParse( """ @@ -354,6 +395,7 @@ final class RegexLiteralTests: XCTestCase { ] ) } + func testClosingSpace2() { assertParse( """ @@ -364,6 +406,7 @@ final class RegexLiteralTests: XCTestCase { ] ) } + func testClosingSpace3() { assertParse( """ @@ -371,6 +414,7 @@ final class RegexLiteralTests: XCTestCase { """ ) } + func testOpeningAndClosingSpace1() { assertParse( """ @@ -381,16 +425,18 @@ final class RegexLiteralTests: XCTestCase { ] ) } + func testOpeningAndClosingSpace2() { assertParse( """ - x += / 1️⃣ / + x += /1️⃣ / """, diagnostics: [ - DiagnosticSpec(message: "bare slash regex literal may not end with space") + DiagnosticSpec(message: "bare slash regex literal may not start with space") ] ) } + func testOpeningAndClosingSpace3() { assertParse( """ @@ -398,6 +444,7 @@ final class RegexLiteralTests: XCTestCase { """ ) } + func testOpeningAndClosingSpace4() { assertParse( """ @@ -408,16 +455,18 @@ final class RegexLiteralTests: XCTestCase { ] ) } + func testOpeningAndClosingSpace5() { assertParse( """ let x = /1️⃣ / """, diagnostics: [ - DiagnosticSpec(message: "bare slash regex literal may not end with space") + DiagnosticSpec(message: "bare slash regex literal may not start with space") ] ) } + func testOpeningAndClosingSpace6() { assertParse( """ @@ -425,11 +474,12 @@ final class RegexLiteralTests: XCTestCase { """ ) } + func testSingleLineTabChar() { - // We currently only keep track of one lexer error, so only diagnose the second. + // We currently only keep track of one lexer error, so only diagnose the first. assertParse( """ - #/\t1️⃣\t/# + #/1️⃣\t\t/# """, diagnostics: [ DiagnosticSpec(message: "unprintable ASCII character found in source file") @@ -444,6 +494,7 @@ final class RegexLiteralTests: XCTestCase { """ ) } + func testBinOpDisambiguation2() { assertParse( """ @@ -451,6 +502,7 @@ final class RegexLiteralTests: XCTestCase { """ ) } + func testBinOpDisambiguation3() { assertParse( """ @@ -458,6 +510,7 @@ final class RegexLiteralTests: XCTestCase { """ ) } + func testBinOpDisambiguation4() { assertParse( """ @@ -465,6 +518,7 @@ final class RegexLiteralTests: XCTestCase { """ ) } + func testBinOpDisambiguation5() { assertParse( """ @@ -472,6 +526,7 @@ final class RegexLiteralTests: XCTestCase { """ ) } + func testBinOpDisambiguation6() { assertParse( """ @@ -479,6 +534,7 @@ final class RegexLiteralTests: XCTestCase { """ ) } + func testBinOpDisambiguation7() { assertParse( """ @@ -486,6 +542,7 @@ final class RegexLiteralTests: XCTestCase { """ ) } + func testBinOpDisambiguation8() { assertParse( """ @@ -493,6 +550,7 @@ final class RegexLiteralTests: XCTestCase { """ ) } + func testBinOpDisambiguation9() { assertParse( """ @@ -500,6 +558,7 @@ final class RegexLiteralTests: XCTestCase { """ ) } + func testBinOpDisambiguation10() { assertParse( """ @@ -507,6 +566,7 @@ final class RegexLiteralTests: XCTestCase { """ ) } + func testBinOpDisambiguation11() { assertParse( """ @@ -514,6 +574,7 @@ final class RegexLiteralTests: XCTestCase { """ ) } + func testBinOpDisambiguation12() { assertParse( """ @@ -521,6 +582,7 @@ final class RegexLiteralTests: XCTestCase { """ ) } + func testBinOpDisambiguation13() { assertParse( """ @@ -528,6 +590,7 @@ final class RegexLiteralTests: XCTestCase { """ ) } + func testBinOpDisambiguation14() { assertParse( """ @@ -535,6 +598,7 @@ final class RegexLiteralTests: XCTestCase { """ ) } + func testBinOpDisambiguation15() { assertParse( """ @@ -542,6 +606,7 @@ final class RegexLiteralTests: XCTestCase { """ ) } + func testBinOpDisambiguation16() { assertParse( """ @@ -549,6 +614,7 @@ final class RegexLiteralTests: XCTestCase { """ ) } + func testBinOpDisambiguation17() { assertParse( """ @@ -556,6 +622,7 @@ final class RegexLiteralTests: XCTestCase { """ ) } + func testBinOpDisambiguation18() { assertParse( """ @@ -563,6 +630,7 @@ final class RegexLiteralTests: XCTestCase { """ ) } + func testBinOpDisambiguation19() { assertParse( """ @@ -570,6 +638,7 @@ final class RegexLiteralTests: XCTestCase { """ ) } + func testBinOpDisambiguation20() { assertParse( """ @@ -577,6 +646,7 @@ final class RegexLiteralTests: XCTestCase { """ ) } + func testBinOpDisambiguation21() { assertParse( """ @@ -584,6 +654,7 @@ final class RegexLiteralTests: XCTestCase { """ ) } + func testBinOpDisambiguation22() { assertParse( """ @@ -591,6 +662,7 @@ final class RegexLiteralTests: XCTestCase { """ ) } + func testBinOpDisambiguation23() { assertParse( """ @@ -598,6 +670,7 @@ final class RegexLiteralTests: XCTestCase { """ ) } + func testBinOpDisambiguation24() { assertParse( """ @@ -605,6 +678,7 @@ final class RegexLiteralTests: XCTestCase { """ ) } + func testBinOpDisambiguation25() { assertParse( """ @@ -615,6 +689,7 @@ final class RegexLiteralTests: XCTestCase { ] ) } + func testBinOpDisambiguation26() { // FIXME: The diagnostic should be one character back assertParse( @@ -626,6 +701,7 @@ final class RegexLiteralTests: XCTestCase { ] ) } + func testBinOpDisambiguation27() { assertParse( """ @@ -633,6 +709,7 @@ final class RegexLiteralTests: XCTestCase { """ ) } + func testBinOpDisambiguation28() { // FIXME: The diagnostic should be one character back assertParse( @@ -644,6 +721,7 @@ final class RegexLiteralTests: XCTestCase { ] ) } + func testBinOpDisambiguation29() { assertParse( """ @@ -651,6 +729,7 @@ final class RegexLiteralTests: XCTestCase { """ ) } + func testBinOpDisambiguation30() { assertParse( """ @@ -658,6 +737,7 @@ final class RegexLiteralTests: XCTestCase { """ ) } + func testBinOpDisambiguation31() { assertParse( """ @@ -665,6 +745,7 @@ final class RegexLiteralTests: XCTestCase { """ ) } + func testBinOpDisambiguation32() { assertParse( """ @@ -672,6 +753,7 @@ final class RegexLiteralTests: XCTestCase { """ ) } + func testBinOpDisambiguation33() { assertParse( """ @@ -679,6 +761,7 @@ final class RegexLiteralTests: XCTestCase { """ ) } + func testBinOpDisambiguation34() { assertParse( """ @@ -686,6 +769,7 @@ final class RegexLiteralTests: XCTestCase { """ ) } + func testBinOpDisambiguation35() { assertParse( """ @@ -693,6 +777,7 @@ final class RegexLiteralTests: XCTestCase { """ ) } + func testBinOpDisambiguation36() { assertParse( """ @@ -700,6 +785,7 @@ final class RegexLiteralTests: XCTestCase { """ ) } + func testBinOpDisambiguation37() { assertParse( """ @@ -707,6 +793,7 @@ final class RegexLiteralTests: XCTestCase { """ ) } + func testBinOpDisambiguation38() { assertParse( """ @@ -714,6 +801,7 @@ final class RegexLiteralTests: XCTestCase { """ ) } + func testBinOpDisambiguation39() { assertParse( """ @@ -721,6 +809,7 @@ final class RegexLiteralTests: XCTestCase { """ ) } + func testBinOpDisambiguation40() { assertParse( """ @@ -728,6 +817,7 @@ final class RegexLiteralTests: XCTestCase { """ ) } + func testBinOpDisambiguation41() { // await is a contextual keyword, so we can't assume it must be a regex. assertParse( @@ -739,6 +829,7 @@ final class RegexLiteralTests: XCTestCase { ] ) } + func testBinOpDisambiguation42() { // await is a contextual keyword, so we can't assume it must be a regex. assertParse( @@ -747,6 +838,7 @@ final class RegexLiteralTests: XCTestCase { """ ) } + func testBinOpDisambiguation43() { assertParse( """ @@ -754,6 +846,7 @@ final class RegexLiteralTests: XCTestCase { """ ) } + func testBinOpDisambiguation44() { assertParse( """ @@ -761,6 +854,7 @@ final class RegexLiteralTests: XCTestCase { """ ) } + func testBinOpDisambiguation45() { assertParse( """ @@ -768,6 +862,7 @@ final class RegexLiteralTests: XCTestCase { """ ) } + func testBinOpDisambiguation46() { assertParse( """ @@ -775,6 +870,7 @@ final class RegexLiteralTests: XCTestCase { """ ) } + func testBinOpDisambiguation47() { assertParse( """ @@ -783,6 +879,7 @@ final class RegexLiteralTests: XCTestCase { """ ) } + func testBinOpDisambiguation48() { assertParse( """ @@ -793,6 +890,7 @@ final class RegexLiteralTests: XCTestCase { """ ) } + func testBinOpDisambiguation49() { assertParse( """ @@ -802,6 +900,7 @@ final class RegexLiteralTests: XCTestCase { """ ) } + func testBinOpDisambiguation50() { assertParse( """ @@ -820,6 +919,7 @@ final class RegexLiteralTests: XCTestCase { ] ) } + func testEmoji() { assertParse("/👍/") } diff --git a/Tests/SwiftParserTest/translated/ForwardSlashRegexSkippingAllowedTests.swift b/Tests/SwiftParserTest/translated/ForwardSlashRegexSkippingAllowedTests.swift index 2d1cad195f6..d1ec81bae2c 100644 --- a/Tests/SwiftParserTest/translated/ForwardSlashRegexSkippingAllowedTests.swift +++ b/Tests/SwiftParserTest/translated/ForwardSlashRegexSkippingAllowedTests.swift @@ -1,31 +1,24 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2023 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// + // This test file has been translated from swift/test/StringProcessing/Parse/forward-slash-regex-skipping-allowed.swift import XCTest final class ForwardSlashRegexSkippingAllowedTests: XCTestCase { - func testForwardSlashRegexSkippingAllowed1() { - assertParse( - """ - // Make sure we can skip in all of the below cases. - """ - ) - } - - func testForwardSlashRegexSkippingAllowed2() { - assertParse( - #""" - // The printing implementation differs in asserts and no-asserts builds, it will - // either print `"Parse.NumFunctionsParsed" 0` or not print it at all. Make sure - // we don't output any non-zero value. - // CHECK-NOT: {{"Parse.NumFunctionsParsed" [^0]}} - """# - ) - } - func testForwardSlashRegexSkippingAllowed3() { + // Ensures there is a parse error assertParse( """ - // Ensures there is a parse error var 1️⃣: Int """, diagnostics: [ @@ -35,9 +28,9 @@ final class ForwardSlashRegexSkippingAllowedTests: XCTestCase { } func testForwardSlashRegexSkippingAllowed4() { + // Balanced `{}`, so okay. assertParse( """ - // Balanced `{}`, so okay. func a() { 1️⃣/ {}/ } """, diagnostics: [ @@ -69,9 +62,9 @@ final class ForwardSlashRegexSkippingAllowedTests: XCTestCase { } func testForwardSlashRegexSkippingAllowed7() { + // Some cases of infix '/' that we should continue to skip. assertParse( """ - // Some cases of infix '/' that we should continue to skip. func d() { _ = 1 / 2 + 3 * 4 _ = 1 / 2 / 3 / 4 @@ -94,26 +87,15 @@ final class ForwardSlashRegexSkippingAllowedTests: XCTestCase { } func testForwardSlashRegexSkippingAllowed9() { + // Some cases of prefix '/' that we should continue to skip. assertParse( """ - // Some cases of prefix '/' that we should continue to skip. prefix operator / prefix func / (_ x: T) -> T { x } """ ) } - func testForwardSlashRegexSkippingAllowed10() { - assertParse( - """ - enum E { - case e - func foo(_ x: T) {} - } - """ - ) - } - func testForwardSlashRegexSkippingAllowed11() { assertParse( """ @@ -141,9 +123,9 @@ final class ForwardSlashRegexSkippingAllowedTests: XCTestCase { } func testForwardSlashRegexSkippingAllowed13() { + // Some cases of postfix '/' that we should continue to skip. assertParse( """ - // Some cases of postfix '/' that we should continue to skip. func g() { _ = 0/ _ = 0/ / 1/ @@ -153,5 +135,4 @@ final class ForwardSlashRegexSkippingAllowedTests: XCTestCase { """ ) } - } diff --git a/Tests/SwiftParserTest/translated/ForwardSlashRegexSkippingInvalidTests.swift b/Tests/SwiftParserTest/translated/ForwardSlashRegexSkippingInvalidTests.swift index 74dc16f52ec..f864b5d1f57 100644 --- a/Tests/SwiftParserTest/translated/ForwardSlashRegexSkippingInvalidTests.swift +++ b/Tests/SwiftParserTest/translated/ForwardSlashRegexSkippingInvalidTests.swift @@ -1,13 +1,25 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2023 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// + // This test file has been translated from swift/test/StringProcessing/Parse/forward-slash-regex-skipping-invalid.swift import XCTest final class ForwardSlashRegexSkippingInvalidTests: XCTestCase { func testForwardSlashRegexSkippingInvalid1() { + // We don't consider this a regex literal when skipping as it has an initial + // space. assertParse( """ - // We don't consider this a regex literal when skipping as it has an initial - // space. func a() { _ = /1️⃣ x*/ } """, diagnostics: [ @@ -17,9 +29,9 @@ final class ForwardSlashRegexSkippingInvalidTests: XCTestCase { } func testForwardSlashRegexSkippingInvalid2() { + // Same because of unbalanced ')' assertParse( """ - // Same because of unbalanced ')' func b() { _ = /x1️⃣)*/ } """, diagnostics: [ @@ -29,9 +41,9 @@ final class ForwardSlashRegexSkippingInvalidTests: XCTestCase { } func testForwardSlashRegexSkippingInvalid3() { + // These also fail the heuristic, but have unbalanced `{` `}`, so we don't skip. assertParse( """ - // These also fail the heuristic, but have unbalanced `{` `}`, so we don't skip. func c() { _ = /1️⃣ x}*/ } func d() { _ = /2️⃣ x{*/ } """, @@ -43,9 +55,9 @@ final class ForwardSlashRegexSkippingInvalidTests: XCTestCase { } func testForwardSlashRegexSkippingInvalid4() { + // Unterminated, and unbalanced `{}`. assertParse( """ - // Unterminated, and unbalanced `{}`. func e() { _ = /1️⃣ }2️⃣ } @@ -191,14 +203,7 @@ final class ForwardSlashRegexSkippingInvalidTests: XCTestCase { } """, diagnostics: [ - // TODO: Old parser expected error on line 4: extraneous '}' at top level - // TODO: Old parser expected error on line 4: consecutive statements on a line must be separated by ';' - // TODO: Old parser expected error on line 4: unterminated regex literal - // TODO: Old parser expected warning on line 4: regular expression literal is unused DiagnosticSpec(message: "extraneous code at top level") - // TODO: Old parser expected warning on line 5: integer literal is unused - // TODO: Old parser expected error on line 6: extraneous '}' at top level - // TODO: Old parser expected error on line 7: extraneous '}' at top level ] ) } @@ -213,10 +218,6 @@ final class ForwardSlashRegexSkippingInvalidTests: XCTestCase { } """, diagnostics: [ - // TODO: Old parser expected error on line 3: consecutive statements on a line must be separated by ';' - // TODO: Old parser expected error on line 3: unterminated regex literal - // TODO: Old parser expected error on line 4: value of type 'Regex' has no member 'bitWidth' - // TODO: Old parser expected error on line 5: extraneous '}' at top level DiagnosticSpec(message: "extraneous code at top level") ] ) @@ -276,5 +277,4 @@ final class ForwardSlashRegexSkippingInvalidTests: XCTestCase { ] ) } - } diff --git a/Tests/SwiftParserTest/translated/ForwardSlashRegexSkippingTests.swift b/Tests/SwiftParserTest/translated/ForwardSlashRegexSkippingTests.swift index a8384cee2eb..a6f22540eb8 100644 --- a/Tests/SwiftParserTest/translated/ForwardSlashRegexSkippingTests.swift +++ b/Tests/SwiftParserTest/translated/ForwardSlashRegexSkippingTests.swift @@ -1,3 +1,15 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2023 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// + // This test file has been translated from swift/test/StringProcessing/Parse/forward-slash-regex-skipping.swift import XCTest @@ -317,7 +329,7 @@ final class ForwardSlashRegexSkippingTests: XCTestCase { """, diagnostics: [ // TODO: Old parser had a fix-it to add backslash to escape - DiagnosticSpec(message: "bare slash regex literal may not end with space") + DiagnosticSpec(message: "bare slash regex literal may not start with space") ] ) } @@ -339,9 +351,9 @@ final class ForwardSlashRegexSkippingTests: XCTestCase { } func testForwardSlashRegexSkipping43() { + // Make sure we don't emit errors for these. assertParse( """ - // Make sure we don't emit errors for these. func err1() { _ = /0xG/ } """ ) diff --git a/Tests/SwiftParserTest/translated/ForwardSlashRegexTests.swift b/Tests/SwiftParserTest/translated/ForwardSlashRegexTests.swift index d3c1cf704b1..04e826802aa 100644 --- a/Tests/SwiftParserTest/translated/ForwardSlashRegexTests.swift +++ b/Tests/SwiftParserTest/translated/ForwardSlashRegexTests.swift @@ -1,5 +1,19 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2023 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// + // This test file has been translated from swift/test/StringProcessing/Parse/forward-slash-regex.swift +@_spi(RawSyntax) import SwiftSyntax +@_spi(RawSyntax) import SwiftParser import XCTest final class ForwardSlashRegexTests: XCTestCase { @@ -22,10 +36,10 @@ final class ForwardSlashRegexTests: XCTestCase { } func testForwardSlashRegex8() { + // The divisions in the body of the below operators make sure we don't try and + // consider them to be ending delimiters of a regex. assertParse( """ - // The divisions in the body of the below operators make sure we don't try and - // consider them to be ending delimiters of a regex. infix operator /^/ : P func /^/ (lhs: Int, rhs: Int) -> Int { 1 / 2 } """ @@ -99,10 +113,10 @@ final class ForwardSlashRegexTests: XCTestCase { } func testForwardSlashRegex18() { + // These unfortunately become prefix `=` and infix `=/` respectively. We could + // likely improve the diagnostic though. assertParse( """ - // These unfortunately become prefix `=` and infix `=/` respectively. We could - // likely improve the diagnostic though. do { let z1️⃣=/0/ } @@ -153,7 +167,7 @@ final class ForwardSlashRegexTests: XCTestCase { _ = !/1️⃣ / """, diagnostics: [ - DiagnosticSpec(message: "bare slash regex literal may not end with space") + DiagnosticSpec(message: "bare slash regex literal may not start with space") ] ) } @@ -164,7 +178,7 @@ final class ForwardSlashRegexTests: XCTestCase { _ = !!/1️⃣ / """, diagnostics: [ - DiagnosticSpec(message: "bare slash regex literal may not end with space") + DiagnosticSpec(message: "bare slash regex literal may not start with space") ] ) } @@ -244,7 +258,8 @@ final class ForwardSlashRegexTests: XCTestCase { _ = /x /? .blah } - """ + """, + substructure: Syntax(BinaryOperatorExprSyntax(operatorToken: .binaryOperator("/?"))) ) } @@ -310,7 +325,6 @@ final class ForwardSlashRegexTests: XCTestCase { } """, diagnostics: [ - // TODO: Old parser expected error on line 3: unary operator cannot be separated from its operand DiagnosticSpec(message: "expected expression after operator"), DiagnosticSpec(message: "unexpected code '?? /x /' in 'do' statement"), ] @@ -318,11 +332,26 @@ final class ForwardSlashRegexTests: XCTestCase { } func testForwardSlashRegex41() { - // This parses as /x/ ??/ x/ + // This parses as /x/?? / x/ assertParse( """ _ = /x/??/x/ - """ + """, + substructure: Syntax( + SequenceExprSyntax( + elements: .init([ + DiscardAssignmentExprSyntax(), + AssignmentExprSyntax(), + OptionalChainingExprSyntax( + expression: OptionalChainingExprSyntax( + expression: RegexLiteralExprSyntax(regexPattern: .regexLiteralPattern("x")) + ) + ), + BinaryOperatorExprSyntax(operatorToken: .binaryOperator("/")), + PostfixUnaryExprSyntax(expression: IdentifierExprSyntax(identifier: "x"), operatorToken: .postfixOperator("/")), + ]) + ) + ) ) } @@ -339,7 +368,8 @@ final class ForwardSlashRegexTests: XCTestCase { assertParse( """ _ = /x/.../y/ - """ + """, + substructure: Syntax(BinaryOperatorExprSyntax(operatorToken: .binaryOperator(".../"))) ) } @@ -360,7 +390,6 @@ final class ForwardSlashRegexTests: XCTestCase { } """, diagnostics: [ - // TODO: Old parser expected error on line 2: operator with postfix spacing cannot start a subexpression DiagnosticSpec(locationMarker: "1️⃣", message: "consecutive statements on a line must be separated by ';'"), DiagnosticSpec(locationMarker: "2️⃣", message: "expected expression in operator"), DiagnosticSpec(locationMarker: "2️⃣", message: "unexpected code '...' in 'do' statement"), @@ -397,14 +426,6 @@ final class ForwardSlashRegexTests: XCTestCase { ) } - func testForwardSlashRegex49() { - assertParse( - """ - func foo(_ x: T, y: T) {} - """ - ) - } - func testForwardSlashRegex50() { assertParse( """ @@ -432,17 +453,6 @@ final class ForwardSlashRegexTests: XCTestCase { ) } - func testForwardSlashRegex54() { - assertParse( - """ - struct S { - subscript(x: Regex) -> Void { () } - subscript(fn: (Int, Int) -> Int) -> Int { 0 } - } - """ - ) - } - func testForwardSlashRegex55() { assertParse( """ @@ -515,7 +525,19 @@ final class ForwardSlashRegexTests: XCTestCase { assertParse( """ _ = [/abc/:/abc/] - """ + """, + substructure: Syntax( + DictionaryExprSyntax( + content: .elements( + .init([ + .init( + keyExpression: RegexLiteralExprSyntax(regexPattern: .regexLiteralPattern("abc")), + valueExpression: RegexLiteralExprSyntax(regexPattern: .regexLiteralPattern("abc")) + ) + ]) + ) + ) + ) ) } @@ -530,7 +552,7 @@ final class ForwardSlashRegexTests: XCTestCase { func testForwardSlashRegex62() { assertParse( """ - _ = [/abc/:/abc/] + _ = [/abc/ :/abc/] """ ) } @@ -611,7 +633,8 @@ final class ForwardSlashRegexTests: XCTestCase { /1 / 2 } - """ + """, + substructure: Syntax(BinaryOperatorExprSyntax(operatorToken: .binaryOperator("/"))) ) } @@ -633,7 +656,8 @@ final class ForwardSlashRegexTests: XCTestCase { """ _ = 2 / 1 / .bitWidth - """ + """, + substructure: Syntax(BinaryOperatorExprSyntax(operatorToken: .binaryOperator("/"))) ) } @@ -643,7 +667,8 @@ final class ForwardSlashRegexTests: XCTestCase { """ _ = 2 /1/ .bitWidth - """ + """, + substructure: Syntax(RegexLiteralExprSyntax(regexPattern: .regexLiteralPattern("1"))) ) } @@ -654,7 +679,8 @@ final class ForwardSlashRegexTests: XCTestCase { _ = 2 / 1 / .bitWidth - """ + """, + substructure: Syntax(BinaryOperatorExprSyntax(operatorToken: .binaryOperator("/"))) ) } @@ -665,7 +691,8 @@ final class ForwardSlashRegexTests: XCTestCase { _ = 2 /1 / .bitWidth - """ + """, + substructure: Syntax(BinaryOperatorExprSyntax(operatorToken: .binaryOperator("/"))) ) } @@ -674,7 +701,16 @@ final class ForwardSlashRegexTests: XCTestCase { assertParse( """ _ = !!/1/ .bitWidth - """ + """, + substructure: Syntax( + PrefixOperatorExprSyntax( + operatorToken: .prefixOperator("!!"), + postfixExpression: MemberAccessExprSyntax( + base: RegexLiteralExprSyntax(regexPattern: .regexLiteralPattern("1")), + name: "bitWidth" + ) + ) + ) ) } @@ -683,7 +719,8 @@ final class ForwardSlashRegexTests: XCTestCase { assertParse( """ _ = !!/1 / .bitWidth - """ + """, + substructure: Syntax(BinaryOperatorExprSyntax(operatorToken: .binaryOperator("/"))) ) } @@ -692,7 +729,8 @@ final class ForwardSlashRegexTests: XCTestCase { """ let z = /y/ - """ + """, + substructure: Syntax(RegexLiteralExprSyntax(regexPattern: .regexLiteralPattern("y"))) ) } @@ -886,7 +924,8 @@ final class ForwardSlashRegexTests: XCTestCase { assertParse( """ _ = /x// comment - """ + """, + substructure: Syntax(PrefixOperatorExprSyntax(operatorToken: .prefixOperator("/"), postfixExpression: IdentifierExprSyntax(identifier: "x"))) ) } @@ -894,7 +933,8 @@ final class ForwardSlashRegexTests: XCTestCase { assertParse( """ _ = /x // comment - """ + """, + substructure: Syntax(PrefixOperatorExprSyntax(operatorToken: .prefixOperator("/"), postfixExpression: IdentifierExprSyntax(identifier: "x"))) ) } @@ -902,7 +942,8 @@ final class ForwardSlashRegexTests: XCTestCase { assertParse( """ _ = /x/*comment*/ - """ + """, + substructure: Syntax(PrefixOperatorExprSyntax(operatorToken: .prefixOperator("/"), postfixExpression: IdentifierExprSyntax(identifier: "x"))) ) } @@ -911,8 +952,19 @@ final class ForwardSlashRegexTests: XCTestCase { assertParse( """ baz(/, /) - baz(/,/) + """, + substructure: Syntax( + TupleExprElementListSyntax([ + .init(expression: IdentifierExprSyntax(identifier: .binaryOperator("/")), trailingComma: .commaToken()), + .init(expression: IdentifierExprSyntax(identifier: .binaryOperator("/"))), + ]) + ) + ) + assertParse( """ + baz(/,/) + """, + substructure: Syntax(RegexLiteralExprSyntax(regexPattern: .regexLiteralPattern(","))) ) } @@ -920,7 +972,20 @@ final class ForwardSlashRegexTests: XCTestCase { assertParse( """ baz((/), /) - """ + """, + substructure: Syntax( + TupleExprElementListSyntax([ + .init( + expression: TupleExprSyntax( + elementList: .init([ + .init(expression: IdentifierExprSyntax(identifier: .binaryOperator("/"))) + ]) + ), + trailingComma: .commaToken() + ), + .init(expression: IdentifierExprSyntax(identifier: .binaryOperator("/"))), + ]) + ) ) } @@ -928,8 +993,19 @@ final class ForwardSlashRegexTests: XCTestCase { assertParse( """ baz(/^, /) - baz(/^,/) + """, + substructure: Syntax( + TupleExprElementListSyntax([ + .init(expression: IdentifierExprSyntax(identifier: .binaryOperator("/^")), trailingComma: .commaToken()), + .init(expression: IdentifierExprSyntax(identifier: .binaryOperator("/"))), + ]) + ) + ) + assertParse( """ + baz(/^,/) + """, + substructure: Syntax(RegexLiteralExprSyntax(regexPattern: .regexLiteralPattern("^,"))) ) } @@ -937,7 +1013,20 @@ final class ForwardSlashRegexTests: XCTestCase { assertParse( """ baz((/^), /) - """ + """, + substructure: Syntax( + TupleExprElementListSyntax([ + .init( + expression: TupleExprSyntax( + elementList: .init([ + .init(expression: IdentifierExprSyntax(identifier: .binaryOperator("/^"))) + ]) + ), + trailingComma: .commaToken() + ), + .init(expression: IdentifierExprSyntax(identifier: .binaryOperator("/"))), + ]) + ) ) } @@ -1166,12 +1255,16 @@ final class ForwardSlashRegexTests: XCTestCase { assertParse( #""" do { - let _: Regex = (/whatever\)/1️⃣ + let _: Regex = ℹ️(/whatever\)/1️⃣ } """#, diagnostics: [ - // TODO: Old parser expected note on line 2: to match this opening '(' - DiagnosticSpec(message: "expected ')' to end tuple") + DiagnosticSpec( + message: "expected ')' to end tuple", + notes: [ + NoteSpec(message: "to match this opening '('") + ] + ) ] ) } @@ -1295,7 +1388,7 @@ final class ForwardSlashRegexTests: XCTestCase { _ = /1️⃣ / """, diagnostics: [ - DiagnosticSpec(message: "bare slash regex literal may not end with space") + DiagnosticSpec(message: "bare slash regex literal may not start with space") ] ) } @@ -1303,10 +1396,10 @@ final class ForwardSlashRegexTests: XCTestCase { func testForwardSlashRegex159() { assertParse( """ - _ = / 1️⃣ / + _ = /1️⃣ / """, diagnostics: [ - DiagnosticSpec(message: "bare slash regex literal may not end with space") + DiagnosticSpec(message: "bare slash regex literal may not start with space") ] ) } @@ -1543,5 +1636,4 @@ final class ForwardSlashRegexTests: XCTestCase { """ ) } - } diff --git a/Tests/SwiftParserTest/translated/PrefixSlashTests.swift b/Tests/SwiftParserTest/translated/PrefixSlashTests.swift index 9e331dfedfe..c4b5195d67f 100644 --- a/Tests/SwiftParserTest/translated/PrefixSlashTests.swift +++ b/Tests/SwiftParserTest/translated/PrefixSlashTests.swift @@ -1,16 +1,20 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2023 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// + // This test file has been translated from swift/test/StringProcessing/Parse/prefix-slash.swift import XCTest final class PrefixSlashTests: XCTestCase { - func testPrefixSlash1() { - assertParse( - """ - // Test the behavior of prefix '/' with regex literals enabled. - """ - ) - } - func testPrefixSlash2() { assertParse( """ @@ -20,17 +24,6 @@ final class PrefixSlashTests: XCTestCase { ) } - func testPrefixSlash3() { - assertParse( - """ - enum E { - case e - func foo(_ x: T) {} - } - """ - ) - } - func testPrefixSlash4() { assertParse( """ @@ -40,14 +33,6 @@ final class PrefixSlashTests: XCTestCase { ) } - func testPrefixSlash5() { - assertParse( - """ - func foo(_ x: T, _ y: U) {} - """ - ) - } - func testPrefixSlash6() { assertParse( """ @@ -58,14 +43,6 @@ final class PrefixSlashTests: XCTestCase { ) } - func testPrefixSlash7() { - assertParse( - """ - func bar(_ x: T) -> Int { 0 } - """ - ) - } - func testPrefixSlash8() { assertParse( """ @@ -73,5 +50,4 @@ final class PrefixSlashTests: XCTestCase { """ ) } - } diff --git a/Tests/SwiftParserTest/translated/RegexParseEndOfBufferTests.swift b/Tests/SwiftParserTest/translated/RegexParseEndOfBufferTests.swift index 073e660cb5b..5929db521f2 100644 --- a/Tests/SwiftParserTest/translated/RegexParseEndOfBufferTests.swift +++ b/Tests/SwiftParserTest/translated/RegexParseEndOfBufferTests.swift @@ -1,3 +1,15 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2023 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// + // This test file has been translated from swift/test/StringProcessing/Parse/regex_parse_end_of_buffer.swift import XCTest @@ -11,5 +23,4 @@ final class RegexParseEndOfBufferTests: XCTestCase { ] ) } - } diff --git a/Tests/SwiftParserTest/translated/RegexParseErrorTests.swift b/Tests/SwiftParserTest/translated/RegexParseErrorTests.swift index b172af0ecfe..92c102d8249 100644 --- a/Tests/SwiftParserTest/translated/RegexParseErrorTests.swift +++ b/Tests/SwiftParserTest/translated/RegexParseErrorTests.swift @@ -1,3 +1,15 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2023 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// + // This test file has been translated from swift/test/StringProcessing/Parse/regex_parse_error.swift import XCTest @@ -162,16 +174,6 @@ final class RegexParseErrorTests: XCTestCase { ) } - func testRegexParseError18() { - assertParse( - """ - func foo( - _ x: T, - _ y: T) {} - """ - ) - } - func testRegexParseError19() { assertParse( """ diff --git a/Tests/SwiftParserTest/translated/RegexTests.swift b/Tests/SwiftParserTest/translated/RegexTests.swift index 18465f89099..0c8f2715a82 100644 --- a/Tests/SwiftParserTest/translated/RegexTests.swift +++ b/Tests/SwiftParserTest/translated/RegexTests.swift @@ -1,3 +1,15 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2023 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// + // This test file has been translated from swift/test/StringProcessing/Parse/regex.swift import XCTest @@ -13,14 +25,6 @@ final class RegexTests: XCTestCase { ) } - func testRegex2() { - assertParse( - """ - func foo(_ x: T...) {} - """ - ) - } - func testRegex3() { assertParse( """ From 7286a77210b0f63e474b7351ce0c77881a42d82f Mon Sep 17 00:00:00 2001 From: Hamish Knight Date: Fri, 31 Mar 2023 10:25:31 +0100 Subject: [PATCH 10/12] Correctly classify operators split from a regex literal Previously we always produced `.prefixOperator`, which is wrong for cases like prefix `&`, `=`, and `?` which should produce different token kinds. Factor out the classification code, and use it to produce the correct token kind. --- Sources/SwiftParser/Lexer/Cursor.swift | 147 ++++++++++-------- .../SwiftParser/Lexer/RegexLiteralLexer.swift | 8 +- Tests/SwiftParserTest/RegexLiteralTests.swift | 67 ++++++++ .../translated/ForwardSlashRegexTests.swift | 9 +- 4 files changed, 162 insertions(+), 69 deletions(-) diff --git a/Sources/SwiftParser/Lexer/Cursor.swift b/Sources/SwiftParser/Lexer/Cursor.swift index bb2e231d732..6497ac19456 100644 --- a/Sources/SwiftParser/Lexer/Cursor.swift +++ b/Sources/SwiftParser/Lexer/Cursor.swift @@ -2029,6 +2029,80 @@ extension Lexer.Cursor { return Lexer.Result(kind, stateTransition: transition) } + /// Classify an operator token given its start and ending cursor. + static func classifyOperatorToken( + operStart: Lexer.Cursor, + operEnd: Lexer.Cursor, + sourceBufferStart: Lexer.Cursor + ) -> (RawTokenKind, error: LexingDiagnostic?) { + // Decide between the binary, prefix, and postfix cases. + // It's binary if either both sides are bound or both sides are not bound. + // Otherwise, it's postfix if left-bound and prefix if right-bound. + let leftBound = operStart.isLeftBound(sourceBufferStart: sourceBufferStart) + let rightBound = operEnd.isRightBound(isLeftBound: leftBound) + + // Match various reserved words. + if operEnd.input.baseAddress! - operStart.input.baseAddress! == 1 { + switch operStart.peek() { + case UInt8(ascii: "="): + if leftBound != rightBound { + var errorPos = operStart + + if rightBound { + _ = errorPos.advance() + } + + return ( + .equal, + error: LexingDiagnostic( + .equalMustHaveConsistentWhitespaceOnBothSides, + position: errorPos + ) + ) + } else { + return (.equal, error: nil) + } + case UInt8(ascii: "&"): + if leftBound == rightBound || leftBound { + break + } + return (.prefixAmpersand, error: nil) + case UInt8(ascii: "."): + return (.period, error: nil) + case UInt8(ascii: "?"): + if (leftBound) { + return (.postfixQuestionMark, error: nil) + } + return (.infixQuestionMark, error: nil) + default: + break + } + } else if (operEnd.input.baseAddress! - operStart.input.baseAddress! == 2) { + switch (operStart.peek(), operStart.peek(at: 1)) { + case (UInt8(ascii: "-"), UInt8(ascii: ">")): // -> + return (.arrow, error: nil) + case (UInt8(ascii: "*"), UInt8(ascii: "/")): // */ + return (.unknown, error: LexingDiagnostic(.unexpectedBlockCommentEnd, position: operStart)) + default: + break + } + } else { + // Verify there is no "*/" in the middle of the identifier token, we reject + // it as potentially ending a block comment. + if operStart.text(upTo: operEnd).contains("*/") { + return (.unknown, error: LexingDiagnostic(.unexpectedBlockCommentEnd, position: operStart)) + } + } + + if leftBound == rightBound { + return (.binaryOperator, error: nil) + } else if leftBound { + return (.postfixOperator, error: nil) + } else { + return (.prefixOperator, error: nil) + } + } + mutating func lexOperatorIdentifier( sourceBufferStart: Lexer.Cursor, preferRegexOverBinaryOperator: Bool @@ -2087,73 +2161,12 @@ extension Lexer.Cursor { _ = ptr.advance() } } - - // Decide between the binary, prefix, and postfix cases. - // It's binary if either both sides are bound or both sides are not bound. - // Otherwise, it's postfix if left-bound and prefix if right-bound. - let leftBound = tokStart.isLeftBound(sourceBufferStart: sourceBufferStart) - let rightBound = self.isRightBound(isLeftBound: leftBound) - - // Match various reserved words. - if self.input.baseAddress! - tokStart.input.baseAddress! == 1 { - switch tokStart.peek() { - case UInt8(ascii: "="): - if leftBound != rightBound { - var errorPos = tokStart - - if rightBound { - _ = errorPos.advance() - } - - return Lexer.Result( - .equal, - error: LexingDiagnostic( - .equalMustHaveConsistentWhitespaceOnBothSides, - position: errorPos - ) - ) - } else { - return Lexer.Result(.equal) - } - case UInt8(ascii: "&"): - if leftBound == rightBound || leftBound { - break - } - return Lexer.Result(.prefixAmpersand) - case UInt8(ascii: "."): - return Lexer.Result(.period) - case UInt8(ascii: "?"): - if (leftBound) { - return Lexer.Result(.postfixQuestionMark) - } - return Lexer.Result(.infixQuestionMark) - default: - break - } - } else if (self.input.baseAddress! - tokStart.input.baseAddress! == 2) { - switch (tokStart.peek(), tokStart.peek(at: 1)) { - case (UInt8(ascii: "-"), UInt8(ascii: ">")): // -> - return Lexer.Result(.arrow) - case (UInt8(ascii: "*"), UInt8(ascii: "/")): // */ - return Lexer.Result(.unknown, error: LexingDiagnostic(.unexpectedBlockCommentEnd, position: tokStart)) - default: - break - } - } else { - // Verify there is no "*/" in the middle of the identifier token, we reject - // it as potentially ending a block comment. - if tokStart.text(upTo: self).contains("*/") { - return Lexer.Result(.unknown, error: LexingDiagnostic(.unexpectedBlockCommentEnd, position: tokStart)) - } - } - - if leftBound == rightBound { - return Lexer.Result(.binaryOperator) - } else if leftBound { - return Lexer.Result(.postfixOperator) - } else { - return Lexer.Result(.prefixOperator) - } + let (kind, error) = Self.classifyOperatorToken( + operStart: tokStart, + operEnd: self, + sourceBufferStart: sourceBufferStart + ) + return Lexer.Result(kind, error: error) } mutating func lexDollarIdentifier() -> Lexer.Result { diff --git a/Sources/SwiftParser/Lexer/RegexLiteralLexer.swift b/Sources/SwiftParser/Lexer/RegexLiteralLexer.swift index 5d8a67058c7..617cc313e48 100644 --- a/Sources/SwiftParser/Lexer/RegexLiteralLexer.swift +++ b/Sources/SwiftParser/Lexer/RegexLiteralLexer.swift @@ -766,8 +766,14 @@ extension Lexer.Cursor { // If we started lexing in the middle of an operator, split off the prefix // operator, and move the cursor to where the regex literal starts. self.position = regexStart.position + let (kind, error) = Self.classifyOperatorToken( + operStart: operatorStart, + operEnd: regexStart, + sourceBufferStart: sourceBufferStart + ) return Lexer.Result( - .prefixOperator, + kind, + error: error, stateTransition: .pushRegexLexemes(index: 0, lexemes: lexemes) ) } else { diff --git a/Tests/SwiftParserTest/RegexLiteralTests.swift b/Tests/SwiftParserTest/RegexLiteralTests.swift index 21eb8ccb855..6d9c0eaf8f9 100644 --- a/Tests/SwiftParserTest/RegexLiteralTests.swift +++ b/Tests/SwiftParserTest/RegexLiteralTests.swift @@ -911,6 +911,73 @@ final class RegexLiteralTests: XCTestCase { ) } + func testPrefixOpSplitting1() { + assertParse( + """ + let x =1️⃣/abc/ + """, + diagnostics: [ + DiagnosticSpec(message: "'=' must have consistent whitespace on both sides") + ] + ) + } + + func testPrefixOpSplitting2() { + assertParse( + """ + let x1️⃣ .2️⃣/abc/ + """, + diagnostics: [ + DiagnosticSpec(locationMarker: "1️⃣", message: "consecutive statements on a line must be separated by ';'"), + DiagnosticSpec(locationMarker: "2️⃣", message: "expected name in member access"), + ] + ) + } + + func testPrefixOpSplitting3() { + assertParse( + """ + let x = true?/abc/1️⃣:/def/ + """, + substructure: Syntax(BinaryOperatorExprSyntax(operatorToken: .binaryOperator("/"))), + diagnostics: [ + DiagnosticSpec(message: "extraneous code ':/def/' at top level") + ] + ) + } + + func testPrefixOpSplitting4() { + assertParse( + """ + let x = true ?/abc/ : /def/ + """, + substructure: Syntax( + SequenceExprSyntax( + elements: .init([ + BooleanLiteralExprSyntax(booleanLiteral: true), + UnresolvedTernaryExprSyntax(firstChoice: RegexLiteralExprSyntax(regexPattern: .regexLiteralPattern("abc"))), + RegexLiteralExprSyntax(regexPattern: .regexLiteralPattern("def")), + ]) + ) + ) + ) + } + + func testPrefixOpSplitting5() { + assertParse( + """ + let x = &/abc/ + """, + substructure: Syntax( + InOutExprSyntax( + expression: RegexLiteralExprSyntax( + regexPattern: .regexLiteralPattern("abc") + ) + ) + ) + ) + } + func testNulCharacter() { assertParse( "/1️⃣\0/", diff --git a/Tests/SwiftParserTest/translated/ForwardSlashRegexTests.swift b/Tests/SwiftParserTest/translated/ForwardSlashRegexTests.swift index 04e826802aa..1d381c7b2a2 100644 --- a/Tests/SwiftParserTest/translated/ForwardSlashRegexTests.swift +++ b/Tests/SwiftParserTest/translated/ForwardSlashRegexTests.swift @@ -449,7 +449,14 @@ final class ForwardSlashRegexTests: XCTestCase { assertParse( """ bar(&/x/) - """ + """, + substructure: Syntax( + InOutExprSyntax( + expression: RegexLiteralExprSyntax( + regexPattern: .regexLiteralPattern("x") + ) + ) + ) ) } From 0b7f66fe4dc06731fb09481f5790cfd2ef272e6c Mon Sep 17 00:00:00 2001 From: Hamish Knight Date: Thu, 30 Mar 2023 21:07:42 +0100 Subject: [PATCH 11/12] Better recover invalid regex in expression position We can confidently lex unless we have a previous token that indicates that we're in an argument list. In such a case, an unapplied operator can be parsed, and we want to ensure we maintain that behavior. --- .../SwiftParser/Lexer/RegexLiteralLexer.swift | 25 +- Tests/SwiftParserTest/DeclarationTests.swift | 15 +- Tests/SwiftParserTest/RegexLiteralTests.swift | 231 +++++++++++++++++- ...orwardSlashRegexSkippingAllowedTests.swift | 12 +- ...orwardSlashRegexSkippingInvalidTests.swift | 8 +- 5 files changed, 255 insertions(+), 36 deletions(-) diff --git a/Sources/SwiftParser/Lexer/RegexLiteralLexer.swift b/Sources/SwiftParser/Lexer/RegexLiteralLexer.swift index 617cc313e48..338c9ddd00d 100644 --- a/Sources/SwiftParser/Lexer/RegexLiteralLexer.swift +++ b/Sources/SwiftParser/Lexer/RegexLiteralLexer.swift @@ -232,6 +232,7 @@ fileprivate struct RegexLiteralLexer { } // Try to lex the opening delimiter. + let openSlash = cursor guard cursor.advance(matching: "/") else { return .notARegex } @@ -263,6 +264,10 @@ fileprivate struct RegexLiteralLexer { break } } + if openSlash.previous == UInt8(ascii: "*") { + // End of block comment, not a regex. + return .notARegex + } } // If the delimiter allows multi-line, try skipping over any whitespace to a @@ -719,15 +724,17 @@ extension Lexer.Cursor { if !mustBeRegex && !operatorStart.isInRegexLiteralPosition() { return nil } - // For better recovery, we can confidently lex a regex literal if the - // previous token was a binary operator and we're in a binary operator - // position, as that would otherwise be illegal. - // TODO: We could probably expand this to other token kinds (or rely on - // `isInRegexLiteralPosition`) for better recovery, but we'd need to be - // wary not to interfere with unapplied operator parsing, which are binary - // operators in expression position. For now, this handles the most common - // case. - if self.previousTokenKind?.is(.binaryOperator, .equal) == true { + // For better recovery, we can confidently lex a regex literal if we're in + // regex literal position, and the '/' is part of what looks like a binary + // operator. This would otherwise be illegal code, as binary operators + // cannot appear in expression position. The only exception to this is if + // the previous token indicates we're in an argument list, in which case + // an unapplied operator is legal, and we should prefer to lex as that + // instead. + switch previousTokenKind { + case .leftParen, .leftSquareBracket, .comma, .colon: + break + default: mustBeRegex = true } } diff --git a/Tests/SwiftParserTest/DeclarationTests.swift b/Tests/SwiftParserTest/DeclarationTests.swift index 1bd21917857..d49f15d23fc 100644 --- a/Tests/SwiftParserTest/DeclarationTests.swift +++ b/Tests/SwiftParserTest/DeclarationTests.swift @@ -741,16 +741,15 @@ final class DeclarationTests: XCTestCase { func testExpressionMember() { assertParse( """ - struct S { - 1️⃣/ 2️⃣#3️⃣#4️⃣#line 5️⃣25 "line-directive.swift" - } + struct S {1️⃣ + /2️⃣ ###line 25 "line-directive.swift"3️⃣ + 4️⃣} """, diagnostics: [ - DiagnosticSpec(locationMarker: "1️⃣", message: "expected 'func' in function"), - DiagnosticSpec(locationMarker: "2️⃣", message: "expected parameter clause in function signature"), - DiagnosticSpec(locationMarker: "3️⃣", message: "expected identifier in macro expansion"), - DiagnosticSpec(locationMarker: "4️⃣", message: "expected identifier in macro expansion"), - DiagnosticSpec(locationMarker: "5️⃣", message: #"unexpected code '25 "line-directive.swift"' in struct"#), + DiagnosticSpec(locationMarker: "1️⃣", message: "expected '}' to end struct"), + DiagnosticSpec(locationMarker: "2️⃣", message: "bare slash regex literal may not start with space"), + DiagnosticSpec(locationMarker: "3️⃣", message: "expected '/' to end regex literal"), + DiagnosticSpec(locationMarker: "4️⃣", message: "extraneous brace at top level"), ] ) } diff --git a/Tests/SwiftParserTest/RegexLiteralTests.swift b/Tests/SwiftParserTest/RegexLiteralTests.swift index 6d9c0eaf8f9..acc3b690812 100644 --- a/Tests/SwiftParserTest/RegexLiteralTests.swift +++ b/Tests/SwiftParserTest/RegexLiteralTests.swift @@ -72,10 +72,10 @@ final class RegexLiteralTests: XCTestCase { func testUnterminated2() { assertParse( #""" - 1️⃣/ + /1️⃣ """#, diagnostics: [ - DiagnosticSpec(message: "extraneous code '/' at top level") + DiagnosticSpec(message: "expected '/' to end regex literal") ] ) } @@ -358,10 +358,10 @@ final class RegexLiteralTests: XCTestCase { func testOpeningSpace1() { assertParse( """ - 1️⃣/ a/ + /1️⃣ a/ """, diagnostics: [ - DiagnosticSpec(message: "extraneous code '/ a/' at top level") + DiagnosticSpec(message: "bare slash regex literal may not start with space") ] ) } @@ -418,10 +418,10 @@ final class RegexLiteralTests: XCTestCase { func testOpeningAndClosingSpace1() { assertParse( """ - 1️⃣/ / + /1️⃣ / """, diagnostics: [ - DiagnosticSpec(message: "extraneous code '/ /' at top level") + DiagnosticSpec(message: "bare slash regex literal may not start with space") ] ) } @@ -448,10 +448,10 @@ final class RegexLiteralTests: XCTestCase { func testOpeningAndClosingSpace4() { assertParse( """ - 1️⃣/ / + /1️⃣ / """, diagnostics: [ - DiagnosticSpec(message: "extraneous code '/ /' at top level") + DiagnosticSpec(message: "bare slash regex literal may not start with space") ] ) } @@ -911,6 +911,221 @@ final class RegexLiteralTests: XCTestCase { ) } + func testBinOpDisambiguation51() { + // Unapplied operators, not regex. + assertParse( + """ + foo(a: /, /) + """, + substructure: Syntax( + TupleExprElementListSyntax([ + .init( + label: "a", + colon: .colonToken(), + expression: IdentifierExprSyntax(identifier: .binaryOperator("/")), + trailingComma: .commaToken() + ), + .init( + expression: IdentifierExprSyntax(identifier: .binaryOperator("/")) + ), + ]) + ) + ) + } + + func testBinOpDisambiguation52() { + // Unapplied operators, not regex. + assertParse( + """ + foo(a, /, /) + """, + substructure: Syntax( + TupleExprElementListSyntax([ + .init( + expression: IdentifierExprSyntax(identifier: "a"), + trailingComma: .commaToken() + ), + .init( + expression: IdentifierExprSyntax(identifier: .binaryOperator("/")), + trailingComma: .commaToken() + ), + .init( + expression: IdentifierExprSyntax(identifier: .binaryOperator("/")) + ), + ]) + ) + ) + } + + func testBinOpDisambiguation53() { + // Unapplied operators, not regex. + assertParse( + """ + foo(a, ^/, /) + """, + substructure: Syntax( + TupleExprElementListSyntax([ + .init( + expression: IdentifierExprSyntax(identifier: "a"), + trailingComma: .commaToken() + ), + .init( + expression: IdentifierExprSyntax(identifier: .binaryOperator("^/")), + trailingComma: .commaToken() + ), + .init( + expression: IdentifierExprSyntax(identifier: .binaryOperator("/")) + ), + ]) + ) + ) + } + + func testBinOpDisambiguation54() { + // Unapplied operators, not regex. + assertParse( + """ + foo(a: ^/, /) + """, + substructure: Syntax( + TupleExprElementListSyntax([ + .init( + label: "a", + colon: .colonToken(), + expression: IdentifierExprSyntax(identifier: .binaryOperator("^/")), + trailingComma: .commaToken() + ), + .init( + expression: IdentifierExprSyntax(identifier: .binaryOperator("/")) + ), + ]) + ) + ) + } + + func testBinOpDisambiguation55() { + // Unapplied operators, not regex. + assertParse( + """ + foo(^/, /) + """, + substructure: Syntax( + TupleExprElementListSyntax([ + .init( + expression: IdentifierExprSyntax(identifier: .binaryOperator("^/")), + trailingComma: .commaToken() + ), + .init( + expression: IdentifierExprSyntax(identifier: .binaryOperator("/")) + ), + ]) + ) + ) + } + + func testBinOpDisambiguation56() { + // Unapplied operators, not regex. + assertParse( + """ + (^/, /) + """, + substructure: Syntax( + TupleExprElementListSyntax([ + .init( + expression: IdentifierExprSyntax(identifier: .binaryOperator("^/")), + trailingComma: .commaToken() + ), + .init( + expression: IdentifierExprSyntax(identifier: .binaryOperator("/")) + ), + ]) + ) + ) + } + + func testBinOpDisambiguation57() { + // Unapplied operators, not regex. + assertParse( + """ + (/, /) + """, + substructure: Syntax( + TupleExprElementListSyntax([ + .init( + expression: IdentifierExprSyntax(identifier: .binaryOperator("/")), + trailingComma: .commaToken() + ), + .init( + expression: IdentifierExprSyntax(identifier: .binaryOperator("/")) + ), + ]) + ) + ) + } + + func testBinOpDisambiguation58() { + // Unapplied operators, not regex. + assertParse( + """ + x[/, /] + """, + substructure: Syntax( + TupleExprElementListSyntax([ + .init( + expression: IdentifierExprSyntax(identifier: .binaryOperator("/")), + trailingComma: .commaToken() + ), + .init( + expression: IdentifierExprSyntax(identifier: .binaryOperator("/")) + ), + ]) + ) + ) + } + + func testBinOpDisambiguation59() { + // Unapplied operators, not regex. + assertParse( + """ + x[^/, /] + """, + substructure: Syntax( + TupleExprElementListSyntax([ + .init( + expression: IdentifierExprSyntax(identifier: .binaryOperator("^/")), + trailingComma: .commaToken() + ), + .init( + expression: IdentifierExprSyntax(identifier: .binaryOperator("/")) + ), + ]) + ) + ) + } + + func testBinOpDisambiguation60() { + // Invalid. We can't confidently lex as a regex (as the lexer thinks it + // could be a subscript), so we get a parser error. + assertParse( + """ + [1️⃣/, /] + """, + diagnostics: [ + DiagnosticSpec(message: "unexpected code '/, /' in array") + ] + ) + } + + func testBinOpDisambiguation61() { + // Fine if there's no trailing space though. + assertParse( + """ + [/,/] + """, + substructure: Syntax(RegexLiteralExprSyntax(regexPattern: .regexLiteralPattern(","))) + ) + } + func testPrefixOpSplitting1() { assertParse( """ diff --git a/Tests/SwiftParserTest/translated/ForwardSlashRegexSkippingAllowedTests.swift b/Tests/SwiftParserTest/translated/ForwardSlashRegexSkippingAllowedTests.swift index d1ec81bae2c..550e60e3a3f 100644 --- a/Tests/SwiftParserTest/translated/ForwardSlashRegexSkippingAllowedTests.swift +++ b/Tests/SwiftParserTest/translated/ForwardSlashRegexSkippingAllowedTests.swift @@ -31,10 +31,10 @@ final class ForwardSlashRegexSkippingAllowedTests: XCTestCase { // Balanced `{}`, so okay. assertParse( """ - func a() { 1️⃣/ {}/ } + func a() { /1️⃣ {}/ } """, diagnostics: [ - DiagnosticSpec(message: "unexpected code '/ {}/' in function") + DiagnosticSpec(message: "bare slash regex literal may not start with space") ] ) } @@ -42,10 +42,10 @@ final class ForwardSlashRegexSkippingAllowedTests: XCTestCase { func testForwardSlashRegexSkippingAllowed5() { assertParse( #""" - func b() { 1️⃣/ \{}/ } + func b() { /1️⃣ \{}/ } """#, diagnostics: [ - DiagnosticSpec(message: #"unexpected code '/ \{}/' in function"#) + DiagnosticSpec(message: "bare slash regex literal may not start with space") ] ) } @@ -53,10 +53,10 @@ final class ForwardSlashRegexSkippingAllowedTests: XCTestCase { func testForwardSlashRegexSkippingAllowed6() { assertParse( #""" - func c() { 1️⃣/ {"{"}/ } + func c() { /1️⃣ {"{"}/ } """#, diagnostics: [ - DiagnosticSpec(message: #"unexpected code '/ {"{"}/' in function"#) + DiagnosticSpec(message: "bare slash regex literal may not start with space") ] ) } diff --git a/Tests/SwiftParserTest/translated/ForwardSlashRegexSkippingInvalidTests.swift b/Tests/SwiftParserTest/translated/ForwardSlashRegexSkippingInvalidTests.swift index f864b5d1f57..58c28998a3b 100644 --- a/Tests/SwiftParserTest/translated/ForwardSlashRegexSkippingInvalidTests.swift +++ b/Tests/SwiftParserTest/translated/ForwardSlashRegexSkippingInvalidTests.swift @@ -178,15 +178,13 @@ final class ForwardSlashRegexSkippingInvalidTests: XCTestCase { } func testForwardSlashRegexSkippingInvalid13() { + // Unbalanced `}`, make sure we don't consider the string literal `{`. assertParse( #""" - // Unbalanced `}`, make sure we don't consider the string literal `{`. - func n() { 2️⃣/ "{"}3️⃣/ } + func n() { /1️⃣ "{"}/ } """#, diagnostics: [ - // TODO: Old parser expected error on line 3: regex literal may not start with space; add backslash to escape - DiagnosticSpec(locationMarker: "2️⃣", message: #"unexpected code '/ "{"' in function"#), - DiagnosticSpec(locationMarker: "3️⃣", message: "extraneous code '/ }' at top level"), + DiagnosticSpec(message: "bare slash regex literal may not start with space") ] ) } From bb0b3eec3003612df78be079d6985c26161e1b0b Mon Sep 17 00:00:00 2001 From: Hamish Knight Date: Fri, 31 Mar 2023 20:36:44 +0100 Subject: [PATCH 12/12] Revert bump pointer allocator slab size change --- Sources/SwiftParser/Lexer/LexemeSequence.swift | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Sources/SwiftParser/Lexer/LexemeSequence.swift b/Sources/SwiftParser/Lexer/LexemeSequence.swift index bac928adfc7..266cbe4db9c 100644 --- a/Sources/SwiftParser/Lexer/LexemeSequence.swift +++ b/Sources/SwiftParser/Lexer/LexemeSequence.swift @@ -29,7 +29,7 @@ extension Lexer { /// /// The memory footpring of not freeing past lexer states is neglible. It's /// usually less than 0.1% of the memory allocated by the syntax arena. - var lexerStateAllocator = BumpPtrAllocator(slabSize: 512) + var lexerStateAllocator = BumpPtrAllocator(slabSize: 256) fileprivate init(sourceBufferStart: Lexer.Cursor, cursor: Lexer.Cursor) { self.sourceBufferStart = sourceBufferStart