Skip to content

Commit fdbc043

Browse files
ahoppenahoppen
committed
Make RawTokenKind a trivial enum
This changes `RawTokenKind` to be a trivial enum again, after I introduced an associated value for `keyword` a few months ago. To avoid repeatedly comparing strings or accessing `Keyword.defaultText` and thus regressing in performance, before matching a lexeme against a bunc of `TokenSpec` that refer to keywords, we pre-compute the keyword it might represent by wrapping it in `PrepareForKeywordMatch`. Performance-wise I have seen slight improvements on on the order of ~2%-3% locally but nothing that I would eyeball as being statistically relevant - but there doesn’t seem to be a performance regression. rdar://104659543
1 parent 43568e9 commit fdbc043

25 files changed

+174
-401
lines changed

‎CodeGeneration/Sources/generate-swiftsyntax/templates/swiftparser/DeclarationModifierFile.swift

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -30,7 +30,7 @@ let declarationModifierFile = SourceFileSyntax {
3030
}
3131

3232
try InitializerDeclSyntax("init?(lexeme: Lexer.Lexeme)") {
33-
try SwitchExprSyntax("switch lexeme") {
33+
try SwitchExprSyntax("switch PrepareForKeywordMatch(lexeme)") {
3434
for attribute in DECL_MODIFIER_KINDS {
3535
SwitchCaseSyntax("case TokenSpec(.\(raw: attribute.swiftName)):") {
3636
ExprSyntax("self = .\(raw: attribute.swiftName)")

‎CodeGeneration/Sources/generate-swiftsyntax/templates/swiftparser/TypeAttributeFile.swift

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -31,7 +31,7 @@ let typeAttributeFile = SourceFileSyntax {
3131
}
3232

3333
try InitializerDeclSyntax("init?(lexeme: Lexer.Lexeme)") {
34-
SwitchExprSyntax(switchKeyword: .keyword(.switch), expression: ExprSyntax("lexeme")) {
34+
try! SwitchExprSyntax("switch PrepareForKeywordMatch(lexeme)") {
3535
for attribute in TYPE_ATTR_KINDS {
3636
SwitchCaseSyntax("case TokenSpec(.\(raw: attribute.name)):") {
3737
ExprSyntax("self = .\(raw: attribute.swiftName)")

‎CodeGeneration/Sources/generate-swiftsyntax/templates/swiftsyntax/TokenKindFile.swift

Lines changed: 14 additions & 81 deletions
Original file line numberDiff line numberDiff line change
@@ -197,101 +197,28 @@ let tokenKindFile = SourceFileSyntax {
197197
// `RawTokenBaseKind` for equality. With the raw value, it compiles down to
198198
// a primitive integer compare, without, it calls into `__derived_enum_equals`.
199199
@frozen // FIXME: Not actually stable, works around a miscompile
200-
public enum RawTokenBaseKind: UInt8, Equatable, Hashable
200+
public enum RawTokenKind: UInt8, Equatable, Hashable
201201
"""
202202
) {
203203
DeclSyntax("case eof")
204204

205205
for token in SYNTAX_TOKENS {
206206
DeclSyntax("case \(raw: token.swiftKind)")
207207
}
208-
}
209-
210-
DeclSyntax(
211-
"""
212-
fileprivate extension Keyword {
213-
static var rawValueZero: Keyword {
214-
return Keyword(rawValue: 0)!
215-
}
216-
}
217-
"""
218-
)
219-
220-
try! StructDeclSyntax(
221-
"""
222-
/// Similar to `TokenKind` but without a `String` associated value.
223-
/// Technically, this should be an enum like
224-
/// ```
225-
/// enum RawTokenKind {
226-
/// case eof
227-
/// case associatedtypeKeyword
228-
/// // remaining case from `RawTokenBaseKind`...
229-
/// case keyword(Keyword)
230-
/// }
231-
/// ```
232-
///
233-
/// But modelling it this way has significant performance implications since
234-
/// comparing two `RawTokenKind` calls into `__derived_enum_equals`. It's more
235-
/// effient to model the base kind as an enum with a raw value and store the
236-
/// keyword separately.
237-
///
238-
/// Whenever `base` is not `keyword`, `keyword` should have a raw value
239-
/// of `0`.
240-
@frozen // FIXME: Not actually stable, works around a miscompile
241-
public struct RawTokenKind: Equatable, Hashable
242-
"""
243-
) {
244-
DeclSyntax("public let base: RawTokenBaseKind")
245-
DeclSyntax("public let keyword: Keyword")
246-
247-
DeclSyntax(
248-
"""
249-
public init(base: RawTokenBaseKind, keyword: Keyword) {
250-
assert(base == .keyword || keyword.rawValue == 0)
251-
self.base = base
252-
self.keyword = keyword
253-
}
254-
"""
255-
)
256-
257-
DeclSyntax(
258-
"""
259-
public static var eof: RawTokenKind {
260-
return RawTokenKind(base: .eof, keyword: .rawValueZero)
261-
}
262-
"""
263-
)
264-
for token in SYNTAX_TOKENS where token.swiftKind != "keyword" {
265-
try VariableDeclSyntax("public static var \(raw: token.swiftKind): RawTokenKind") {
266-
StmtSyntax("return RawTokenKind(base: .\(raw: token.swiftKind), keyword: .rawValueZero)")
267-
}
268-
}
269-
270-
DeclSyntax(
271-
"""
272-
public static func keyword(_ keyword: Keyword) -> RawTokenKind {
273-
return RawTokenKind(base: .keyword, keyword: keyword)
274-
}
275-
"""
276-
)
277208

278209
try VariableDeclSyntax(
279210
"""
280211
@_spi(RawSyntax)
281212
public var defaultText: SyntaxText?
282213
"""
283214
) {
284-
try! SwitchExprSyntax("switch self.base") {
215+
try! SwitchExprSyntax("switch self") {
285216
SwitchCaseSyntax("case .eof:") {
286217
StmtSyntax(#"return """#)
287218
}
288219

289220
for token in SYNTAX_TOKENS {
290-
if token.swiftKind == "keyword" {
291-
SwitchCaseSyntax("case .\(raw: token.swiftKind):") {
292-
StmtSyntax("return self.keyword.defaultText")
293-
}
294-
} else if let text = token.text {
221+
if let text = token.text {
295222
SwitchCaseSyntax("case .\(raw: token.swiftKind):") {
296223
StmtSyntax("return #\"\(raw: text)\"#")
297224
}
@@ -314,7 +241,7 @@ let tokenKindFile = SourceFileSyntax {
314241
public var isPunctuation: Bool
315242
"""
316243
) {
317-
try! SwitchExprSyntax("switch self.base") {
244+
try! SwitchExprSyntax("switch self") {
318245
SwitchCaseSyntax("case .eof:") {
319246
StmtSyntax("return false")
320247
}
@@ -336,16 +263,22 @@ let tokenKindFile = SourceFileSyntax {
336263
public static func fromRaw(kind rawKind: RawTokenKind, text: String) -> TokenKind
337264
"""
338265
) {
339-
try! SwitchExprSyntax("switch rawKind.base") {
266+
try! SwitchExprSyntax("switch rawKind") {
340267
SwitchCaseSyntax("case .eof:") {
341268
StmtSyntax("return .eof")
342269
}
343270

344271
for token in SYNTAX_TOKENS {
345272
if token.swiftKind == "keyword" {
346273
SwitchCaseSyntax("case .\(raw: token.swiftKind):") {
347-
ExprSyntax("assert(text.isEmpty || String(syntaxText: rawKind.keyword.defaultText) == text)")
348-
StmtSyntax("return .keyword(rawKind.keyword)")
274+
DeclSyntax("var text = text")
275+
StmtSyntax(
276+
"""
277+
return text.withSyntaxText { text in
278+
return .keyword(Keyword(text)!)
279+
}
280+
"""
281+
)
349282
}
350283
} else if token.text != nil {
351284
SwitchCaseSyntax("case .\(raw: token.swiftKind):") {
@@ -377,7 +310,7 @@ let tokenKindFile = SourceFileSyntax {
377310
for token in SYNTAX_TOKENS {
378311
if token.swiftKind == "keyword" {
379312
SwitchCaseSyntax("case .\(raw: token.swiftKind)(let keyword):") {
380-
StmtSyntax("return (.\(raw: token.swiftKind)(keyword), nil)")
313+
StmtSyntax("return (.\(raw: token.swiftKind), String(syntaxText: keyword.defaultText))")
381314
}
382315
} else if token.text != nil {
383316
SwitchCaseSyntax("case .\(raw: token.swiftKind):") {

‎Sources/SwiftParser/Attributes.swift

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -65,7 +65,7 @@ extension Parser {
6565
case transpose
6666

6767
init?(lexeme: Lexer.Lexeme) {
68-
switch lexeme {
68+
switch PrepareForKeywordMatch(lexeme) {
6969
case TokenSpec(._alignment): self = ._alignment
7070
case TokenSpec(._backDeploy): self = ._backDeploy
7171
case TokenSpec(._cdecl): self = ._cdecl
@@ -366,7 +366,7 @@ extension Parser {
366366
case forward
367367

368368
init?(lexeme: Lexer.Lexeme) {
369-
switch lexeme {
369+
switch PrepareForKeywordMatch(lexeme) {
370370
case TokenSpec(.reverse): self = .reverse
371371
case TokenSpec(._linear): self = .linear
372372
case TokenSpec(._forward): self = .forward
@@ -477,7 +477,7 @@ extension Parser {
477477
case selfKeyword
478478

479479
init?(lexeme: Lexer.Lexeme) {
480-
switch lexeme {
480+
switch PrepareForKeywordMatch(lexeme) {
481481
case TokenSpec(.identifier): self = .identifier
482482
case TokenSpec(.integerLiteral): self = .integerLiteral
483483
case TokenSpec(.self): self = .selfKeyword
@@ -661,7 +661,7 @@ extension Parser {
661661
case available
662662

663663
init?(lexeme: Lexer.Lexeme) {
664-
switch lexeme {
664+
switch PrepareForKeywordMatch(lexeme) {
665665
case TokenSpec(.target): self = .target
666666
case TokenSpec(.availability): self = .availability
667667
case TokenSpec(.exported): self = .exported
@@ -1102,7 +1102,7 @@ extension Parser {
11021102
}
11031103

11041104
init?(lexeme: Lexer.Lexeme) {
1105-
switch lexeme {
1105+
switch PrepareForKeywordMatch(lexeme) {
11061106
case TokenSpec(.private): self = .private
11071107
case TokenSpec(.fileprivate): self = .fileprivate
11081108
case TokenSpec(.internal): self = .internal

‎Sources/SwiftParser/Availability.swift

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -66,7 +66,7 @@ extension Parser {
6666
case identifier
6767

6868
init?(lexeme: Lexer.Lexeme) {
69-
switch lexeme {
69+
switch PrepareForKeywordMatch(lexeme) {
7070
case TokenSpec(.message): self = .message
7171
case TokenSpec(.renamed): self = .renamed
7272
case TokenSpec(.introduced): self = .introduced

‎Sources/SwiftParser/Declarations.swift

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -345,7 +345,7 @@ extension Parser {
345345
}
346346

347347
init?(lexeme: Lexer.Lexeme) {
348-
switch lexeme {
348+
switch PrepareForKeywordMatch(lexeme) {
349349
case TokenSpec(.typealias): self = .typealias
350350
case TokenSpec(.struct): self = .struct
351351
case TokenSpec(.class): self = .class
@@ -556,7 +556,7 @@ extension Parser {
556556
case nativeClassLayout
557557

558558
init?(lexeme: Lexer.Lexeme) {
559-
switch lexeme {
559+
switch PrepareForKeywordMatch(lexeme) {
560560
case TokenSpec(._Trivial): self = .trivialLayout
561561
case TokenSpec(._TrivialAtMost): self = .trivialAtMostLayout
562562
case TokenSpec(._UnknownLayout): self = .unknownLayout
@@ -1494,7 +1494,7 @@ extension Parser {
14941494
if hasTryBeforeIntroducer && !value.is(RawTryExprSyntax.self) {
14951495
value = RawExprSyntax(
14961496
RawTryExprSyntax(
1497-
tryKeyword: missingToken(.keyword(.try), text: nil),
1497+
tryKeyword: missingToken(.try),
14981498
questionOrExclamationMark: nil,
14991499
expression: value,
15001500
arena: self.arena
@@ -1993,7 +1993,7 @@ extension Parser {
19931993
case lowerThan
19941994

19951995
init?(lexeme: Lexer.Lexeme) {
1996-
switch lexeme {
1996+
switch PrepareForKeywordMatch(lexeme) {
19971997
case TokenSpec(.associativity): self = .associativity
19981998
case TokenSpec(.assignment): self = .assignment
19991999
case TokenSpec(.higherThan): self = .higherThan

‎Sources/SwiftParser/Directives.swift

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -27,7 +27,7 @@ extension Parser {
2727
}
2828

2929
init?(lexeme: Lexer.Lexeme) {
30-
switch lexeme {
30+
switch PrepareForKeywordMatch(lexeme) {
3131
case TokenSpec(.poundIfKeyword): self = .poundIfKeyword
3232
case TokenSpec(.poundElseifKeyword): self = .poundElseifKeyword
3333
case TokenSpec(.poundElseKeyword): self = .poundElseKeyword

‎Sources/SwiftParser/Expressions.swift

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -266,7 +266,7 @@ extension Parser {
266266
case throwsKeyword
267267

268268
init?(lexeme: Lexer.Lexeme) {
269-
switch lexeme {
269+
switch PrepareForKeywordMatch(lexeme) {
270270
case TokenSpec(.binaryOperator): self = .binaryOperator
271271
case TokenSpec(.infixQuestionMark): self = .infixQuestionMark
272272
case TokenSpec(.equal): self = .equal
@@ -2256,7 +2256,7 @@ extension Parser {
22562256
unknownAttr: nil,
22572257
label: .case(
22582258
RawSwitchCaseLabelSyntax(
2259-
caseKeyword: missingToken(.keyword(.case), text: nil),
2259+
caseKeyword: missingToken(.case),
22602260
caseItems: RawCaseItemListSyntax(
22612261
elements: [
22622262
RawCaseItemSyntax(

‎Sources/SwiftParser/Lexer/Cursor.swift

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -218,7 +218,7 @@ extension Lexer {
218218
var input: UnsafeBufferPointer<UInt8>
219219
var previous: UInt8
220220
/// If we have already lexed a token, the kind of the previously lexed token
221-
var previousTokenKind: RawTokenBaseKind?
221+
var previousTokenKind: RawTokenKind?
222222
private var stateStack: StateStack = StateStack()
223223

224224
init(input: UnsafeBufferPointer<UInt8>, previous: UInt8) {
@@ -357,7 +357,7 @@ extension Lexer.Cursor {
357357
flags.insert(.isAtStartOfLine)
358358
}
359359

360-
self.previousTokenKind = result.tokenKind.base
360+
self.previousTokenKind = result.tokenKind
361361
diagnostic = TokenDiagnostic(combining: diagnostic, result.error?.tokenDiagnostic(tokenStart: cursor))
362362

363363
return .init(
@@ -1900,7 +1900,7 @@ extension Lexer.Cursor {
19001900

19011901
let text = tokStart.text(upTo: self)
19021902
if let keyword = Keyword(text), keyword.isLexerClassified {
1903-
return Lexer.Result(.keyword(keyword))
1903+
return Lexer.Result(.keyword)
19041904
} else if text == "_" {
19051905
return Lexer.Result(.wildcard)
19061906
} else {

‎Sources/SwiftParser/Modifiers.swift

Lines changed: 2 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -166,7 +166,7 @@ extension Parser {
166166
}
167167

168168
init?(lexeme: Lexer.Lexeme) {
169-
switch lexeme {
169+
switch PrepareForKeywordMatch(lexeme) {
170170
case TokenSpec(.private): self = .private
171171
case TokenSpec(.fileprivate): self = .fileprivate
172172
case TokenSpec(.internal): self = .internal
@@ -200,11 +200,7 @@ extension Parser {
200200
(unexpectedBeforeDetail, detail) = eat(setHandle)
201201
} else {
202202
unexpectedBeforeDetail = nil
203-
detail = RawTokenSyntax(
204-
missing: .keyword(.set),
205-
text: "set",
206-
arena: arena
207-
)
203+
detail = missingToken(.set)
208204
}
209205
let (unexpectedBeforeRightParen, rightParen) = expect(.rightParen)
210206

‎Sources/SwiftParser/Names.swift

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -282,7 +282,7 @@ extension Lexer.Lexeme {
282282
// Only lexer-classified lexemes have `RawTokenKind` of `keyword.
283283
// Contextual keywords will only be made keywords when a `RawTokenSyntax` is
284284
// constructed from them.
285-
return self.rawTokenKind.base == .keyword
285+
return self.rawTokenKind == .keyword
286286
}
287287

288288
func starts(with symbol: SyntaxText) -> Bool {

‎Sources/SwiftParser/Parser.swift

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -169,6 +169,11 @@ public struct Parser {
169169
return RawTokenSyntax(missing: kind, text: text, arena: self.arena)
170170
}
171171

172+
@_spi(RawSyntax)
173+
public mutating func missingToken(_ keyword: Keyword) -> RawTokenSyntax {
174+
return missingToken(.keyword, text: keyword.defaultText)
175+
}
176+
172177
/// Consumes the current token and advances the lexer to the next token.
173178
///
174179
/// - Returns: The token that was consumed.

‎Sources/SwiftParser/Patterns.swift

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -55,7 +55,7 @@ extension Parser {
5555
case varKeyword
5656

5757
init?(lexeme: Lexer.Lexeme) {
58-
switch lexeme {
58+
switch PrepareForKeywordMatch(lexeme) {
5959
case TokenSpec(.leftParen): self = .leftParen
6060
case TokenSpec(.wildcard): self = .wildcard
6161
case TokenSpec(.identifier): self = .identifier
@@ -296,7 +296,7 @@ extension Parser.Lookahead {
296296
case leftParen
297297

298298
init?(lexeme: Lexer.Lexeme) {
299-
switch lexeme {
299+
switch PrepareForKeywordMatch(lexeme) {
300300
case TokenSpec(.identifier): self = .identifier
301301
case TokenSpec(.wildcard): self = .wildcard
302302
case TokenSpec(.let): self = .letKeyword

‎Sources/SwiftParser/Recovery.swift

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -92,7 +92,7 @@ extension Parser.Lookahead {
9292
tokenConsumptionHandle: TokenConsumptionHandle(spec: matchedSpec)
9393
)
9494
}
95-
let currentTokenPrecedence = TokenPrecedence(self.currentToken.rawTokenKind)
95+
let currentTokenPrecedence = TokenPrecedence(self.currentToken)
9696
if currentTokenPrecedence >= recoveryPrecedence {
9797
break
9898
}
@@ -141,7 +141,7 @@ extension Parser.Lookahead {
141141
)
142142
)
143143
}
144-
let currentTokenPrecedence = TokenPrecedence(self.currentToken.rawTokenKind)
144+
let currentTokenPrecedence = TokenPrecedence(self.currentToken)
145145
if currentTokenPrecedence >= recoveryPrecedence {
146146
break
147147
}

0 commit comments

Comments
 (0)