Skip to content

Validate token choices in CI #1466

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 1 commit into from
Apr 13, 2023
Merged

Validate token choices in CI #1466

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 7 additions & 4 deletions Package.swift
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,13 +5,13 @@ import Foundation

/// If we are in a controlled CI environment, we can use internal compiler flags
/// to speed up the build or improve it.
let swiftSyntaxSwiftSettings: [SwiftSetting]
var swiftSyntaxSwiftSettings: [SwiftSetting] = []
if ProcessInfo.processInfo.environment["SWIFT_BUILD_SCRIPT_ENVIRONMENT"] != nil {
let groupFile = URL(fileURLWithPath: #file)
.deletingLastPathComponent()
.appendingPathComponent("utils")
.appendingPathComponent("group.json")
swiftSyntaxSwiftSettings = [
swiftSyntaxSwiftSettings += [
.define("SWIFTSYNTAX_ENABLE_ASSERTIONS"),
.unsafeFlags([
"-Xfrontend", "-group-info-path",
Expand All @@ -21,8 +21,11 @@ if ProcessInfo.processInfo.environment["SWIFT_BUILD_SCRIPT_ENVIRONMENT"] != nil
"-enforce-exclusivity=unchecked",
]),
]
} else {
swiftSyntaxSwiftSettings = []
}
if ProcessInfo.processInfo.environment["SWIFTSYNTAX_ENABLE_RAWSYNTAX_VALIDATION"] != nil {
swiftSyntaxSwiftSettings += [
.define("SWIFTSYNTAX_ENABLE_RAWSYNTAX_VALIDATION")
]
}

let package = Package(
Expand Down
10 changes: 1 addition & 9 deletions Sources/SwiftSyntax/generated/raw/RawSyntaxValidation.swift
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@
/// Note that this only validates the immediate children.
/// Results in an assertion failure if the layout is invalid.
func validateLayout(layout: RawSyntaxBuffer, as kind: SyntaxKind) {
#if DEBUG
#if SWIFTSYNTAX_ENABLE_RAWSYNTAX_VALIDATION
enum TokenChoice: CustomStringConvertible {
case keyword(StaticString)
case tokenKind(RawTokenKind)
Expand Down Expand Up @@ -140,7 +140,6 @@ func validateLayout(layout: RawSyntaxBuffer, as kind: SyntaxKind) {
// the list of expected token choices in the syntax tree doesn't match those
// the parser generates. Disable the verification for now until all issues
// regarding it are fixed.
#if VALIDATE_TOKEN_CHOICES
if raw != nil {
return verify(
raw,
Expand All @@ -151,9 +150,6 @@ func validateLayout(layout: RawSyntaxBuffer, as kind: SyntaxKind) {
)
}
return nil
#else
return verify(raw, as: RawTokenSyntax?.self)
#endif
}
func verify(
_ raw: RawSyntax?,
Expand All @@ -166,7 +162,6 @@ func validateLayout(layout: RawSyntaxBuffer, as kind: SyntaxKind) {
// the list of expected token choices in the syntax tree doesn't match those
// the parser generates. Disable the verification for now until all issues
// regarding it are fixed.
#if VALIDATE_TOKEN_CHOICES
guard let raw = raw else {
return .expectedNonNil(expectedKind: RawTokenSyntax.self, file: file, line: line)
}
Expand All @@ -193,9 +188,6 @@ func validateLayout(layout: RawSyntaxBuffer, as kind: SyntaxKind) {
file: file,
line: line
)
#else
return verify(raw, as: RawTokenSyntax.self)
#endif
}
func assertNoError(_ nodeKind: SyntaxKind, _ index: Int, _ error: ValidationError?) {
if let error = error {
Expand Down
33 changes: 28 additions & 5 deletions build-script.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,6 @@
import subprocess
import sys
import tempfile
from concurrent.futures import ThreadPoolExecutor
from typing import Dict, List, Optional


Expand Down Expand Up @@ -141,6 +140,7 @@ def run_code_generation(

env = dict(os.environ)
env["SWIFT_BUILD_SCRIPT_ENVIRONMENT"] = "1"
env["SWIFTSYNTAX_ENABLE_RAWSYNTAX_VALIDATION"] = "1"
check_call(swiftpm_call, env=env, verbose=verbose)


Expand Down Expand Up @@ -174,6 +174,7 @@ class Builder(object):
build_dir: Optional[str]
multiroot_data_file: Optional[str]
release: bool
enable_rawsyntax_validation: bool
disable_sandbox: bool

def __init__(
Expand All @@ -182,12 +183,14 @@ def __init__(
build_dir: Optional[str],
multiroot_data_file: Optional[str],
release: bool,
enable_rawsyntax_validation: bool,
verbose: bool,
disable_sandbox: bool = False,
) -> None:
self.build_dir = build_dir
self.multiroot_data_file = multiroot_data_file
self.release = release
self.enable_rawsyntax_validation = enable_rawsyntax_validation
self.disable_sandbox = disable_sandbox
self.verbose = verbose
self.toolchain = toolchain
Expand Down Expand Up @@ -221,6 +224,8 @@ def __build(self, package_dir: str, product_name: str) -> None:

env = dict(os.environ)
env["SWIFT_BUILD_SCRIPT_ENVIRONMENT"] = "1"
if self.enable_rawsyntax_validation:
env["SWIFTSYNTAX_ENABLE_RAWSYNTAX_VALIDATION"] = "1"
# Tell other projects in the unified build to use local dependencies
env["SWIFTCI_USE_LOCAL_DEPS"] = "1"
env["SWIFT_SYNTAX_PARSER_LIB_SEARCH_PATH"] = \
Expand Down Expand Up @@ -274,7 +279,8 @@ def check_generated_files_match(self_generated_dir: str,

def run_tests(
toolchain: str, build_dir: Optional[str], multiroot_data_file: Optional[str],
release: bool, filecheck_exec: Optional[str], skip_lit_tests: bool, verbose: bool
release: bool, enable_rawsyntax_validation: bool, filecheck_exec: Optional[str],
skip_lit_tests: bool, verbose: bool
) -> None:
print("** Running SwiftSyntax Tests **")

Expand All @@ -292,6 +298,7 @@ def run_tests(
build_dir=build_dir,
multiroot_data_file=multiroot_data_file,
release=release,
enable_rawsyntax_validation=enable_rawsyntax_validation,
verbose=verbose,
)

Expand Down Expand Up @@ -396,6 +403,7 @@ def run_lit_tests(toolchain: str, build_dir: Optional[str], release: bool,

def run_xctests(toolchain: str, build_dir: Optional[str],
multiroot_data_file: Optional[str], release: bool,
enable_rawsyntax_validation: bool,
verbose: bool) -> None:
print("** Running XCTests **")
swiftpm_call = get_swiftpm_invocation(
Expand All @@ -414,6 +422,8 @@ def run_xctests(toolchain: str, build_dir: Optional[str],

env = dict(os.environ)
env["SWIFT_BUILD_SCRIPT_ENVIRONMENT"] = "1"
if enable_rawsyntax_validation:
env["SWIFTSYNTAX_ENABLE_RAWSYNTAX_VALIDATION"] = "1"
# Tell other projects in the unified build to use local dependencies
env["SWIFTCI_USE_LOCAL_DEPS"] = "1"
env["SWIFT_SYNTAX_PARSER_LIB_SEARCH_PATH"] = \
Expand Down Expand Up @@ -461,10 +471,11 @@ def verify_source_code_command(args: argparse.Namespace) -> None:
def build_command(args: argparse.Namespace) -> None:
try:
builder = Builder(
toolchain=realpath(args.toolchain),
toolchain=realpath(args.toolchain), # pyright: ignore
build_dir=realpath(args.build_dir),
multiroot_data_file=args.multiroot_data_file,
release=args.release,
enable_rawsyntax_validation=args.enable_rawsyntax_validation,
verbose=args.verbose,
disable_sandbox=args.disable_sandbox,
)
Expand All @@ -484,10 +495,11 @@ def build_command(args: argparse.Namespace) -> None:
def test_command(args: argparse.Namespace) -> None:
try:
builder = Builder(
toolchain=realpath(args.toolchain),
toolchain=realpath(args.toolchain), # pyright: ignore
build_dir=realpath(args.build_dir),
multiroot_data_file=args.multiroot_data_file,
release=args.release,
enable_rawsyntax_validation=args.enable_rawsyntax_validation,
verbose=args.verbose,
disable_sandbox=args.disable_sandbox,
)
Expand All @@ -496,10 +508,11 @@ def test_command(args: argparse.Namespace) -> None:
builder.buildExample("ExamplePlugin")

run_tests(
toolchain=realpath(args.toolchain),
toolchain=realpath(args.toolchain), # pyright: ignore
build_dir=realpath(args.build_dir),
multiroot_data_file=args.multiroot_data_file,
release=args.release,
enable_rawsyntax_validation=args.enable_rawsyntax_validation,
filecheck_exec=realpath(args.filecheck_exec),
skip_lit_tests=args.skip_lit_tests,
verbose=args.verbose,
Expand Down Expand Up @@ -552,6 +565,16 @@ def add_default_build_arguments(parser: argparse.ArgumentParser) -> None:
""",
)

parser.add_argument(
"--enable-rawsyntax-validation",
action="store_true",
help="""
When constructing RawSyntax nodes validate that their layout matches that
defined in `CodeGeneration` and that TokenSyntax nodes have a `tokenKind`
matching the ones specified in `CodeGeneration`.
"""
)

parser.add_argument(
"--toolchain",
required=True,
Expand Down