Skip to content

Commit e5f7200

Browse files
per1234per1234
committed
Embed schemas in code
Previously, the schemas were provided as external files. This presented some difficulties for testing and was also likely to make distribution and installation more complex. By embedding the schemas as binary data in the Go code, the tool can consist of a single standalone binary.
1 parent 3d8592b commit e5f7200

32 files changed

+2094
-335
lines changed

‎.github/workflows/test.yml

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@ on:
99
- "go.sum"
1010
- "**/*.go"
1111
- "**/testdata/**"
12+
- "etc/schemas/**/*.json"
1213
pull_request:
1314
paths:
1415
- ".github/workflows/test.yml"
@@ -17,6 +18,7 @@ on:
1718
- "go.sum"
1819
- "**/*.go"
1920
- "**/testdata/**"
21+
- "etc/schemas/**/*.json"
2022

2123
jobs:
2224
test-go:
@@ -44,6 +46,12 @@ jobs:
4446
repo-token: ${{ secrets.GITHUB_TOKEN }}
4547
version: 3.x
4648

49+
- name: Generate code
50+
run: task go:generate
51+
52+
- name: Check for forgotten code generation
53+
run: git diff --color --exit-code
54+
4755
- name: Build
4856
run: task build
4957

‎.prettierignore

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@
55
.ionide/
66

77
# Test files
8-
/check/checkdata/schema/testdata/invalid-schema.json
8+
/check/checkdata/schema/testdata/input/invalid-schema.json
99
/check/checkdata/testdata/packageindexes/invalid-JSON/package_foo_index.json
1010
/check/checkfunctions/testdata/packageindexes/invalid-JSON/package_foo_index.json
1111
/check/checkfunctions/testdata/sketches/InvalidJSONMetadataFile/sketch.json

‎Taskfile.yml

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,14 @@ tasks:
1212
- task: go:test-unit
1313
- task: schema:compile
1414

15+
go:generate:
16+
desc: Generate Go code
17+
cmds:
18+
- go get -u "github.com/go-bindata/go-bindata/...@v3.1.1"
19+
- go-bindata -nocompress -nometadata -o "./check/checkdata/schema/schemadata/bindata.go" --pkg schemadata --prefix "./etc/schemas/" "./etc/schemas/"
20+
- go-bindata -nocompress -nometadata -o "./check/checkdata/schema/testdata/bindata.go" --pkg testdata --prefix "./check/checkdata/schema/testdata/input/" "./check/checkdata/schema/testdata/input/"
21+
- task: go:format
22+
1523
go:test-unit:
1624
desc: Run unit tests
1725
cmds:

‎check/check.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -34,7 +34,7 @@ import (
3434
func RunChecks(project project.Type) {
3535
feedback.Printf("\nChecking %s in %s\n", project.ProjectType, project.Path)
3636

37-
checkdata.Initialize(project, configuration.SchemasPath())
37+
checkdata.Initialize(project)
3838

3939
for _, checkConfiguration := range checkconfigurations.Configurations() {
4040
runCheck, err := shouldRun(checkConfiguration, project)

‎check/checkdata/checkdata.go

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -27,15 +27,15 @@ import (
2727
)
2828

2929
// Initialize gathers the check data for the specified project.
30-
func Initialize(project project.Type, schemasPath *paths.Path) {
30+
func Initialize(project project.Type) {
3131
superprojectType = project.SuperprojectType
3232
projectType = project.ProjectType
3333
projectPath = project.Path
3434
switch project.ProjectType {
3535
case projecttype.Sketch:
3636
InitializeForSketch(project)
3737
case projecttype.Library:
38-
InitializeForLibrary(project, schemasPath)
38+
InitializeForLibrary(project)
3939
case projecttype.Platform:
4040
InitializeForPlatform(project)
4141
case projecttype.PackageIndex:

‎check/checkdata/library.go

Lines changed: 5 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -21,20 +21,19 @@ import (
2121
"net/http"
2222
"os"
2323

24+
"github.com/arduino/arduino-check/check/checkdata/schema"
2425
"github.com/arduino/arduino-check/check/checkdata/schema/compliancelevel"
2526
"github.com/arduino/arduino-check/project"
2627
"github.com/arduino/arduino-check/project/library/libraryproperties"
2728
"github.com/arduino/arduino-check/result/feedback"
2829
"github.com/arduino/arduino-cli/arduino/libraries"
29-
"github.com/arduino/go-paths-helper"
3030
"github.com/arduino/go-properties-orderedmap"
3131
"github.com/client9/misspell"
32-
"github.com/ory/jsonschema/v3"
3332
"github.com/sirupsen/logrus"
3433
)
3534

3635
// Initialize gathers the library check data for the specified project.
37-
func InitializeForLibrary(project project.Type, schemasPath *paths.Path) {
36+
func InitializeForLibrary(project project.Type) {
3837
var err error
3938

4039
libraryProperties, libraryPropertiesLoadError = libraryproperties.Properties(project.Path)
@@ -43,7 +42,7 @@ func InitializeForLibrary(project project.Type, schemasPath *paths.Path) {
4342
// TODO: can I even do this?
4443
libraryPropertiesSchemaValidationResult = nil
4544
} else {
46-
libraryPropertiesSchemaValidationResult = libraryproperties.Validate(libraryProperties, schemasPath)
45+
libraryPropertiesSchemaValidationResult = libraryproperties.Validate(libraryProperties)
4746
}
4847

4948
loadedLibrary, err = libraries.Load(project.Path, libraries.User)
@@ -98,10 +97,10 @@ func LibraryProperties() *properties.Map {
9897
return libraryProperties
9998
}
10099

101-
var libraryPropertiesSchemaValidationResult map[compliancelevel.Type]*jsonschema.ValidationError
100+
var libraryPropertiesSchemaValidationResult map[compliancelevel.Type]schema.ValidationResult
102101

103102
// LibraryPropertiesSchemaValidationResult returns the result of validating library.properties against the JSON schema.
104-
func LibraryPropertiesSchemaValidationResult() map[compliancelevel.Type]*jsonschema.ValidationError {
103+
func LibraryPropertiesSchemaValidationResult() map[compliancelevel.Type]schema.ValidationResult {
105104
return libraryPropertiesSchemaValidationResult
106105
}
107106

‎check/checkdata/packageindex_test.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -53,7 +53,7 @@ func TestInitializeForPackageIndex(t *testing.T) {
5353
ProjectType: projecttype.PackageIndex,
5454
SuperprojectType: projecttype.PackageIndex,
5555
}
56-
Initialize(testProject, nil)
56+
Initialize(testProject)
5757

5858
testTable.packageIndexLoadErrorAssertion(t, PackageIndexLoadError(), testTable.testName)
5959
if PackageIndexLoadError() == nil {

‎check/checkdata/platform_test.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -53,7 +53,7 @@ func TestInitializeForPlatform(t *testing.T) {
5353
ProjectType: projecttype.Platform,
5454
SuperprojectType: projecttype.Platform,
5555
}
56-
Initialize(testProject, nil)
56+
Initialize(testProject)
5757

5858
testTable.boardsTxtLoadErrorAssertion(t, BoardsTxtLoadError(), testTable.testName)
5959
if BoardsTxtLoadError() == nil {

‎check/checkdata/schema/parsevalidationresult.go

Lines changed: 35 additions & 38 deletions
Original file line numberDiff line numberDiff line change
@@ -20,39 +20,38 @@ import (
2020
"fmt"
2121
"regexp"
2222

23-
"github.com/arduino/go-paths-helper"
2423
"github.com/ory/jsonschema/v3"
2524
"github.com/sirupsen/logrus"
2625
)
2726

2827
// RequiredPropertyMissing returns whether the given required property is missing from the document.
29-
func RequiredPropertyMissing(propertyName string, validationResult *jsonschema.ValidationError, schemasPath *paths.Path) bool {
30-
return ValidationErrorMatch("#", "/required$", "", "^#/"+propertyName+"$", validationResult, schemasPath)
28+
func RequiredPropertyMissing(propertyName string, validationResult ValidationResult) bool {
29+
return ValidationErrorMatch("#", "/required$", "", "^#/"+propertyName+"$", validationResult)
3130
}
3231

3332
// PropertyPatternMismatch returns whether the given property did not match the regular expression defined in the JSON schema.
34-
func PropertyPatternMismatch(propertyName string, validationResult *jsonschema.ValidationError, schemasPath *paths.Path) bool {
35-
return ValidationErrorMatch("#/"+propertyName, "/pattern$", "", "", validationResult, schemasPath)
33+
func PropertyPatternMismatch(propertyName string, validationResult ValidationResult) bool {
34+
return ValidationErrorMatch("#/"+propertyName, "/pattern$", "", "", validationResult)
3635
}
3736

3837
// PropertyLessThanMinLength returns whether the given property is less than the minimum length allowed by the schema.
39-
func PropertyLessThanMinLength(propertyName string, validationResult *jsonschema.ValidationError, schemasPath *paths.Path) bool {
40-
return ValidationErrorMatch("^#/"+propertyName+"$", "/minLength$", "", "", validationResult, schemasPath)
38+
func PropertyLessThanMinLength(propertyName string, validationResult ValidationResult) bool {
39+
return ValidationErrorMatch("^#/"+propertyName+"$", "/minLength$", "", "", validationResult)
4140
}
4241

4342
// PropertyGreaterThanMaxLength returns whether the given property is greater than the maximum length allowed by the schema.
44-
func PropertyGreaterThanMaxLength(propertyName string, validationResult *jsonschema.ValidationError, schemasPath *paths.Path) bool {
45-
return ValidationErrorMatch("^#/"+propertyName+"$", "/maxLength$", "", "", validationResult, schemasPath)
43+
func PropertyGreaterThanMaxLength(propertyName string, validationResult ValidationResult) bool {
44+
return ValidationErrorMatch("^#/"+propertyName+"$", "/maxLength$", "", "", validationResult)
4645
}
4746

4847
// PropertyEnumMismatch returns whether the given property does not match any of the items in the enum array.
49-
func PropertyEnumMismatch(propertyName string, validationResult *jsonschema.ValidationError, schemasPath *paths.Path) bool {
50-
return ValidationErrorMatch("#/"+propertyName, "/enum$", "", "", validationResult, schemasPath)
48+
func PropertyEnumMismatch(propertyName string, validationResult ValidationResult) bool {
49+
return ValidationErrorMatch("#/"+propertyName, "/enum$", "", "", validationResult)
5150
}
5251

5352
// MisspelledOptionalPropertyFound returns whether a misspelled optional property was found.
54-
func MisspelledOptionalPropertyFound(validationResult *jsonschema.ValidationError, schemasPath *paths.Path) bool {
55-
return ValidationErrorMatch("#/", "/misspelledOptionalProperties/", "", "", validationResult, schemasPath)
53+
func MisspelledOptionalPropertyFound(validationResult ValidationResult) bool {
54+
return ValidationErrorMatch("#/", "/misspelledOptionalProperties/", "", "", validationResult)
5655
}
5756

5857
// ValidationErrorMatch returns whether the given query matches against the JSON schema validation error.
@@ -62,10 +61,9 @@ func ValidationErrorMatch(
6261
schemaPointerQuery,
6362
schemaPointerValueQuery,
6463
failureContextQuery string,
65-
validationResult *jsonschema.ValidationError,
66-
schemasPath *paths.Path,
64+
validationResult ValidationResult,
6765
) bool {
68-
if validationResult == nil {
66+
if validationResult.Result == nil {
6967
// No error, so nothing to match.
7068
logrus.Trace("Schema validation passed. No match is possible.")
7169
return false
@@ -82,28 +80,27 @@ func ValidationErrorMatch(
8280
schemaPointerValueRegexp,
8381
failureContextRegexp,
8482
validationResult,
85-
schemasPath)
83+
)
8684
}
8785

8886
func validationErrorMatch(
8987
instancePointerRegexp,
9088
schemaPointerRegexp,
9189
schemaPointerValueRegexp,
9290
failureContextRegexp *regexp.Regexp,
93-
validationError *jsonschema.ValidationError,
94-
schemasPath *paths.Path,
91+
validationError ValidationResult,
9592
) bool {
9693
logrus.Trace("--------Checking schema validation failure match--------")
97-
logrus.Tracef("Checking instance pointer: %s match with regexp: %s", validationError.InstancePtr, instancePointerRegexp)
98-
if instancePointerRegexp.MatchString(validationError.InstancePtr) {
94+
logrus.Tracef("Checking instance pointer: %s match with regexp: %s", validationError.Result.InstancePtr, instancePointerRegexp)
95+
if instancePointerRegexp.MatchString(validationError.Result.InstancePtr) {
9996
logrus.Tracef("Matched!")
100-
matchedSchemaPointer := validationErrorSchemaPointerMatch(schemaPointerRegexp, validationError, schemasPath)
97+
matchedSchemaPointer := validationErrorSchemaPointerMatch(schemaPointerRegexp, validationError)
10198
if matchedSchemaPointer != "" {
10299
logrus.Tracef("Matched!")
103-
if validationErrorSchemaPointerValueMatch(schemaPointerValueRegexp, validationError.SchemaURL, matchedSchemaPointer, schemasPath) {
100+
if validationErrorSchemaPointerValueMatch(schemaPointerValueRegexp, validationError, matchedSchemaPointer) {
104101
logrus.Tracef("Matched!")
105-
logrus.Tracef("Checking failure context: %v match with regexp: %s", validationError.Context, failureContextRegexp)
106-
if validationErrorContextMatch(failureContextRegexp, validationError) {
102+
logrus.Tracef("Checking failure context: %v match with regexp: %s", validationError.Result.Context, failureContextRegexp)
103+
if validationErrorContextMatch(failureContextRegexp, validationError.Result) {
107104
logrus.Tracef("Matched!")
108105
return true
109106
}
@@ -112,14 +109,16 @@ func validationErrorMatch(
112109
}
113110

114111
// Recursively check all causes for a match.
115-
for _, validationErrorCause := range validationError.Causes {
112+
for _, validationErrorCause := range validationError.Result.Causes {
116113
if validationErrorMatch(
117114
instancePointerRegexp,
118115
schemaPointerRegexp,
119116
schemaPointerValueRegexp,
120117
failureContextRegexp,
121-
validationErrorCause,
122-
schemasPath,
118+
ValidationResult{
119+
Result: validationErrorCause,
120+
dataLoader: validationError.dataLoader,
121+
},
123122
) {
124123
return true
125124
}
@@ -131,18 +130,17 @@ func validationErrorMatch(
131130
// validationErrorSchemaPointerMatch matches the JSON schema pointer related to the validation failure against a regular expression.
132131
func validationErrorSchemaPointerMatch(
133132
schemaPointerRegexp *regexp.Regexp,
134-
validationError *jsonschema.ValidationError,
135-
schemasPath *paths.Path,
133+
validationError ValidationResult,
136134
) string {
137-
logrus.Tracef("Checking schema pointer: %s match with regexp: %s", validationError.SchemaPtr, schemaPointerRegexp)
138-
if schemaPointerRegexp.MatchString(validationError.SchemaPtr) {
139-
return validationError.SchemaPtr
135+
logrus.Tracef("Checking schema pointer: %s match with regexp: %s", validationError.Result.SchemaPtr, schemaPointerRegexp)
136+
if schemaPointerRegexp.MatchString(validationError.Result.SchemaPtr) {
137+
return validationError.Result.SchemaPtr
140138
}
141139

142140
// The schema validator does not provide full pointer past logic inversion keywords to the lowest level keywords related to the validation error cause.
143141
// Therefore, the sub-keywords must be checked for matches in order to be able to interpret the exact cause of the failure.
144-
if regexp.MustCompile("(/not)|(/oneOf)$").MatchString(validationError.SchemaPtr) {
145-
return validationErrorSchemaSubPointerMatch(schemaPointerRegexp, validationError.SchemaPtr, validationErrorSchemaPointerValue(validationError, schemasPath))
142+
if regexp.MustCompile("(/not)|(/oneOf)$").MatchString(validationError.Result.SchemaPtr) {
143+
return validationErrorSchemaSubPointerMatch(schemaPointerRegexp, validationError.Result.SchemaPtr, validationErrorSchemaPointerValue(validationError))
146144
}
147145

148146
return ""
@@ -184,11 +182,10 @@ func validationErrorSchemaSubPointerMatch(schemaPointerRegexp *regexp.Regexp, pa
184182
// it matches against the given regular expression.
185183
func validationErrorSchemaPointerValueMatch(
186184
schemaPointerValueRegexp *regexp.Regexp,
187-
schemaURL,
185+
validationError ValidationResult,
188186
schemaPointer string,
189-
schemasPath *paths.Path,
190187
) bool {
191-
marshalledSchemaPointerValue, err := json.Marshal(schemaPointerValue(schemaURL, schemaPointer, schemasPath))
188+
marshalledSchemaPointerValue, err := json.Marshal(schemaPointerValue(validationError.Result.SchemaURL, schemaPointer, validationError.dataLoader))
192189
if err != nil {
193190
panic(err)
194191
}

0 commit comments

Comments
 (0)