Skip to content

Commit 93e917b

Browse files
StevenWong12StevenWong12
committed
WIP: Compute the next potential token for each node
Add a compute property to represent layout as syntax kinds
1 parent c4f2f47 commit 93e917b

File tree

13 files changed

+6510
-75
lines changed

13 files changed

+6510
-75
lines changed

‎CodeGeneration/Sources/SyntaxSupport/Child.swift

Lines changed: 31 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,7 @@
1212

1313
/// The kind of token a node can contain. Either a token of a specific kind or a
1414
/// keyword with the given text.
15-
public enum TokenChoice {
15+
public enum TokenChoice: CustomStringConvertible {
1616
case keyword(text: String)
1717
case token(tokenKind: String)
1818

@@ -22,6 +22,24 @@ public enum TokenChoice {
2222
case .token: return false
2323
}
2424
}
25+
26+
public var description: String {
27+
switch self {
28+
case .keyword(text: let string):
29+
guard KEYWORDS.first(where: { $0.name == string }) != nil else {
30+
assertionFailure("no such keyword named \(string)")
31+
return ""
32+
}
33+
return ".keyword"
34+
case .token(tokenKind: let string):
35+
guard let token = SYNTAX_TOKENS.first(where: { $0.name == string.components(separatedBy: "Token").first }) else {
36+
assertionFailure("no such token named \(string)")
37+
return ""
38+
}
39+
return ".\(token.swiftKind)"
40+
}
41+
}
42+
2543
}
2644

2745
public enum ChildKind {
@@ -146,6 +164,18 @@ public class Child {
146164
return hasBaseType && isOptional
147165
}
148166

167+
public var childKindChoices: [SyntaxNodeKind] {
168+
if self.hasBaseType == true {
169+
return SYNTAX_NODES.filter { $0.base == self.syntaxNodeKind }.map { $0.kind }
170+
}
171+
172+
if case .nodeChoices(choices: let choices) = self.kind {
173+
return choices.flatMap { $0.childKindChoices }
174+
}
175+
176+
return [self.syntaxNodeKind]
177+
}
178+
149179
/// If a classification is passed, it specifies the color identifiers in
150180
/// that subtree should inherit for syntax coloring. Must be a member of
151181
/// ``SyntaxClassification``.

‎CodeGeneration/Sources/SyntaxSupport/DeclNodes.swift

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -460,7 +460,6 @@ public let DECL_NODES: [Node] = [
460460
.keyword(text: "public"),
461461
.keyword(text: "reasync"),
462462
.keyword(text: "required"),
463-
.keyword(text: "setter_access"),
464463
.keyword(text: "static"),
465464
.keyword(text: "unowned"),
466465
.keyword(text: "weak"),

‎CodeGeneration/Sources/SyntaxSupport/Node.swift

Lines changed: 34 additions & 32 deletions
Original file line numberDiff line numberDiff line change
@@ -84,53 +84,55 @@ public class Node {
8484
return CollectionNode(node: self)
8585
}
8686
}
87-
87+
8888
/// Get the first potential token of a node recursively
8989
/// This information is used by incremental parse
9090
public var firstPotentialToken: [TokenChoice] {
9191
if let children = self.layoutNode?.children,
92-
let firstChild = children.first(where: {!$0.isUnexpectedNodes}) {
93-
92+
let firstChild = children.first(where: { !$0.isUnexpectedNodes })
93+
{
94+
9495
switch firstChild.kind {
95-
case .node(kind: let nodeKind):
96-
guard let nodeSpec = SYNTAX_NODES.first(where: {$0.kind == nodeKind}) else {
97-
return []
96+
case .node(kind: let nodeKind):
97+
guard let nodeSpec = SYNTAX_NODES.first(where: { $0.kind == nodeKind }) else {
98+
return []
99+
}
100+
return nodeSpec.firstPotentialToken
101+
102+
case .nodeChoices(choices: let children):
103+
return children.compactMap { child in
104+
if let nodeSpec = SYNTAX_NODES.first(where: { $0.kind == child.syntaxNodeKind }) {
105+
return nodeSpec.firstPotentialToken
98106
}
99-
return nodeSpec.firstPotentialToken
100-
101-
case .nodeChoices(choices: let children):
102-
return children.compactMap { child in
103-
if let nodeSpec = SYNTAX_NODES.first(where: {$0.kind == child.syntaxNodeKind}) {
104-
return nodeSpec.firstPotentialToken
105-
}
106-
return nil
107-
}.flatMap {$0}
108-
case .collection(kind: let kind, _):
109-
guard let nodeSpec = SYNTAX_NODES.first(where: {$0.kind == kind}),
110-
let collection = nodeSpec.collectionNode else {
111-
return []
107+
return nil
108+
}.flatMap { $0 }
109+
case .collection(kind: let kind, _):
110+
guard let nodeSpec = SYNTAX_NODES.first(where: { $0.kind == kind }),
111+
let collection = nodeSpec.collectionNode
112+
else {
113+
return []
114+
}
115+
return collection.elementChoices.compactMap { choiceKind in
116+
if let nodeSpec = SYNTAX_NODES.first(where: { $0.kind == choiceKind }) {
117+
return nodeSpec.firstPotentialToken
112118
}
113-
return collection.elementChoices.compactMap { choiceKind in
114-
if let nodeSpec = SYNTAX_NODES.first(where: {$0.kind == choiceKind}) {
115-
return nodeSpec.firstPotentialToken
116-
}
117-
return nil
118-
}.flatMap {$0}
119-
120-
case .token(choices: let choices, _, _):
121-
return choices
119+
return nil
120+
}.flatMap { $0 }
121+
122+
case .token(choices: let choices, _, _):
123+
return choices
122124
}
123125
}
124-
126+
125127
if let collection = self.collectionNode {
126128
return collection.elementChoices.compactMap { choiceKind in
127-
if let nodeSpec = SYNTAX_NODES.first(where: {$0.kind == choiceKind}) {
129+
if let nodeSpec = SYNTAX_NODES.first(where: { $0.kind == choiceKind }) {
128130
return nodeSpec.firstPotentialToken
129131
}
130132
return nil
131-
}.flatMap {$0}
133+
}.flatMap { $0 }
132134
}
133-
135+
134136
return []
135137
}
136138

‎CodeGeneration/Sources/SyntaxSupport/SyntaxNodeKind.swift

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,7 @@ import SwiftSyntaxBuilder
1717
///
1818
/// Using the cases of this enum, children of syntax nodes can refer the syntax
1919
/// node that defines their layout.
20-
public enum SyntaxNodeKind: String, CaseIterable {
20+
public enum SyntaxNodeKind: String, CaseIterable, CustomStringConvertible {
2121
// Please keep this list sorted alphabetically
2222

2323
case accessorBlock
@@ -365,4 +365,8 @@ public enum SyntaxNodeKind: String, CaseIterable {
365365
return "Raw\(raw: rawValue.withFirstCharacterUppercased)SyntaxNodeProtocol"
366366
}
367367
}
368+
369+
public var description: String {
370+
return ".\(rawValue)"
371+
}
368372
}

‎CodeGeneration/Sources/generate-swiftsyntax/templates/swiftparser/ParserEntryFile.swift

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -27,7 +27,7 @@ let parserEntryFile = SourceFileSyntax(leadingTrivia: copyrightHeader) {
2727
source: String,
2828
parseTransition: IncrementalParseTransition? = nil
2929
) -> SourceFileSyntax {
30-
var parser = Parser(source)
30+
var parser = Parser(source, parseTransition: parseTransition)
3131
return SourceFileSyntax.parse(from: &parser)
3232
}
3333
"""
@@ -42,7 +42,7 @@ let parserEntryFile = SourceFileSyntax(leadingTrivia: copyrightHeader) {
4242
maximumNestingLevel: Int? = nil,
4343
parseTransition: IncrementalParseTransition? = nil
4444
) -> SourceFileSyntax {
45-
var parser = Parser(source, maximumNestingLevel: maximumNestingLevel)
45+
var parser = Parser(source, maximumNestingLevel: maximumNestingLevel, parseTransition: parseTransition)
4646
return SourceFileSyntax.parse(from: &parser)
4747
}
4848
"""

‎CodeGeneration/Sources/generate-swiftsyntax/templates/swiftsyntax/SyntaxKindFile.swift

Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -68,5 +68,45 @@ let syntaxKindFile = SourceFileSyntax(leadingTrivia: copyrightHeader) {
6868
}
6969
}
7070
}
71+
72+
try VariableDeclSyntax(
73+
"""
74+
@_spi(RawSyntax)
75+
public var firstPotentialTokenKind: [RawTokenKind]
76+
"""
77+
) {
78+
try SwitchExprSyntax("switch self") {
79+
80+
for node in NON_BASE_SYNTAX_NODES {
81+
SwitchCaseSyntax("case .\(node.varOrCaseName):") {
82+
StmtSyntax("return [\(raw: Set(node.firstPotentialToken.map{$0.description}).joined(separator: ", "))]")
83+
}
84+
}
85+
SwitchCaseSyntax("default:") {
86+
StmtSyntax("return []")
87+
}
88+
}
89+
}
90+
91+
try VariableDeclSyntax(
92+
"""
93+
@_spi(RawSyntax)
94+
public var syntaxLayout: [[SyntaxKind]]
95+
"""
96+
) {
97+
try SwitchExprSyntax("switch self") {
98+
99+
for node in NON_BASE_SYNTAX_NODES.filter({ $0.layoutNode != nil }) {
100+
SwitchCaseSyntax("case .\(node.varOrCaseName):") {
101+
if let layoutNode = node.layoutNode {
102+
StmtSyntax("return [\(raw: layoutNode.children.map{$0.childKindChoices.description}.joined(separator: ", "))]")
103+
}
104+
}
105+
}
106+
SwitchCaseSyntax("default:") {
107+
StmtSyntax("return []")
108+
}
109+
}
110+
}
71111
}
72112
}

‎Sources/SwiftParser/Parser.swift

Lines changed: 7 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -634,26 +634,23 @@ extension Parser {
634634
var lookUpHelper = IncrementalParseLookup(transition: parseTransition)
635635
let currentOffset = self.lexemes.getOffsetToStart(self.currentToken)
636636
if let node = lookUpHelper.lookUp(currentOffset, kind: kind),
637-
nodeCanBeReused(node: node) == true {
638-
// for test
637+
nodeCanBeReused(node: node) == true
638+
{
639639
lookUpHelper.registerReusedNode(node, offset: currentOffset)
640640
self.lexemes.advance(by: node.byteSize, currentToken: &self.currentToken)
641641
return node
642642
}
643643

644644
return nil
645645
}
646-
646+
647647
func nodeCanBeReused(node: Syntax) -> Bool {
648648
var lookahead = self.lookahead()
649649
lookahead.lexemes.advance(by: node.byteSize, currentToken: &lookahead.currentToken)
650-
651-
if let codeBlockItem = node.as(CodeBlockItemSyntax.self),
652-
let internalItem = codeBlockItem.item.as(FunctionCallExprSyntax.self),
653-
let nextPotentialToken = internalItem.nextPotentialTokenChoices?.decomposeToRaw().rawKind {
654-
return lookahead.currentToken.rawTokenKind != nextPotentialToken
650+
if let codeBlockItem = node.as(CodeBlockItemSyntax.self) {
651+
return !codeBlockItem.item.raw.nextPotentialTokenChoices.contains(lookahead.currentToken.rawTokenKind)
655652
}
656-
657-
return true
653+
654+
return !node.raw.nextPotentialTokenChoices.contains(lookahead.currentToken.rawTokenKind)
658655
}
659656
}

‎Sources/SwiftSyntax/IncrementalParseTransition.swift

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -242,7 +242,7 @@ public struct IncrementalParseLookup {
242242
let node = cursorLookup(prevPosition: prevPosition, kind: kind)
243243
return node
244244
}
245-
245+
246246
public func registerReusedNode(_ node: Syntax, offset: Int) {
247247
if let delegate = reusedDelegate {
248248
delegate.parserReusedNode(

‎Sources/SwiftSyntax/Raw/RawSyntax.swift

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -917,6 +917,30 @@ extension RawSyntax {
917917
}
918918
}
919919

920+
extension RawSyntax {
921+
public var nextPotentialTokenChoices: [RawTokenKind] {
922+
if let children = self.layoutView?.children,
923+
let lastNonNilChildIdx = children.lastIndex(where: { $0 != nil }),
924+
lastNonNilChildIdx != children.endIndex
925+
{
926+
927+
var nextPotentialNodeIndex = children.distance(from: children.startIndex, to: children.index(after: lastNonNilChildIdx))
928+
929+
if self.kind.syntaxLayout[nextPotentialNodeIndex].allSatisfy({ $0 == .unexpectedNodes }) {
930+
nextPotentialNodeIndex += 1
931+
}
932+
933+
if nextPotentialNodeIndex == children.count {
934+
return []
935+
}
936+
937+
return self.kind.syntaxLayout[nextPotentialNodeIndex].flatMap { $0.firstPotentialTokenKind }
938+
939+
}
940+
return []
941+
}
942+
}
943+
920944
#if DEBUG
921945
/// See `SyntaxMemoryLayout`.
922946
var RawSyntaxDataMemoryLayouts: [String: SyntaxMemoryLayout.Value] = [

0 commit comments

Comments
 (0)