From 0bee5348bd5beea5284e1e70cfddc9101080086a Mon Sep 17 00:00:00 2001 From: Anuraag Agrawal Date: Fri, 1 Dec 2023 16:51:25 +0900 Subject: [PATCH 01/20] Use wazero-based wrapper of libpgquery when cgo is disabled or on windows --- .github/workflows/ci.yml | 17 +++++++-- go.mod | 3 ++ go.sum | 4 +++ internal/codegen/sdk/utils.go | 8 +++-- internal/engine/postgresql/convert.go | 5 +-- internal/engine/postgresql/parse.go | 9 ++--- internal/engine/postgresql/parse_default.go | 35 +++++++++++++++++++ internal/engine/postgresql/parse_disabled.go | 35 ------------------- internal/engine/postgresql/parse_wasi.go | 10 ++++++ .../postgresql/parser/parser_default.go | 8 +++++ .../engine/postgresql/parser/parser_wasi.go | 8 +++++ internal/engine/postgresql/utils.go | 5 +-- 12 files changed, 93 insertions(+), 54 deletions(-) create mode 100644 internal/engine/postgresql/parse_default.go delete mode 100644 internal/engine/postgresql/parse_disabled.go create mode 100644 internal/engine/postgresql/parse_wasi.go create mode 100644 internal/engine/postgresql/parser/parser_default.go create mode 100644 internal/engine/postgresql/parser/parser_wasi.go diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f525c4c31d..681733bae7 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -14,7 +14,7 @@ jobs: with: go-version-file: go.mod check-latest: true - - run: go build ./... + - run: go test ./... darwin-build: if: ${{ github.ref == 'refs/heads/main' }} @@ -25,7 +25,20 @@ jobs: with: go-version-file: go.mod check-latest: true - - run: go build ./... + - run: go test ./... + + linux-purego-build: + if: ${{ github.ref == 'refs/heads/main' }} + runs-on: macos-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-go@v4 + with: + go-version-file: go.mod + check-latest: true + - run: go test ./... + env: + CGO_ENABLED: '0' build: name: test diff --git a/go.mod b/go.mod index fa2da759e6..384024a76e 100644 --- a/go.mod +++ b/go.mod @@ -20,6 +20,7 @@ require ( github.com/riza-io/grpc-go v0.2.0 github.com/spf13/cobra v1.8.0 github.com/spf13/pflag v1.0.5 + github.com/wasilibs/go-pgquery v0.0.0-20231201074715-bbab8b1ab4a9 github.com/xeipuuv/gojsonschema v1.2.0 golang.org/x/sync v0.5.0 google.golang.org/grpc v1.59.0 @@ -27,6 +28,8 @@ require ( gopkg.in/yaml.v3 v3.0.1 ) +require github.com/wasilibs/wazerox v0.0.0-20231117065139-b3503f4aeff6 // indirect + require ( github.com/antlr4-go/antlr/v4 v4.13.0 // indirect github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548 // indirect diff --git a/go.sum b/go.sum index 90d91347c6..106fde3f3f 100644 --- a/go.sum +++ b/go.sum @@ -185,6 +185,10 @@ github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +github.com/wasilibs/go-pgquery v0.0.0-20231201074715-bbab8b1ab4a9 h1:JrDBgtPAsn6LC0EMn9jBaSFiZqVpKMa9hea47/nkaeA= +github.com/wasilibs/go-pgquery v0.0.0-20231201074715-bbab8b1ab4a9/go.mod h1:kalxVxSjXzPXnzVrIsejhYyPa81QLKHbNnNSB8bL6q0= +github.com/wasilibs/wazerox v0.0.0-20231117065139-b3503f4aeff6 h1:jwbU8u5TuXModzdEG4wI0g4FyuD7ROSttU86go5sPdU= +github.com/wasilibs/wazerox v0.0.0-20231117065139-b3503f4aeff6/go.mod h1:IQNVyA4d1hWIe23mlMMuqXjyWMdndgSlNx6FqBkwPsM= github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f h1:J9EGpcZtP0E/raorCMxlFGSTBrsSlaDGf3jU/qvAE2c= github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU= github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 h1:EzJWgHovont7NscjpAxXsDA8S8BMYve8Y5+7cuRE7R0= diff --git a/internal/codegen/sdk/utils.go b/internal/codegen/sdk/utils.go index 1dffda9e7e..ec4a4e5ab4 100644 --- a/internal/codegen/sdk/utils.go +++ b/internal/codegen/sdk/utils.go @@ -23,12 +23,14 @@ func Title(s string) string { // a backtick, replace it the following way: // // input: -// SELECT `group` FROM foo +// +// SELECT `group` FROM foo // // output: -// SELECT ` + "`" + `group` + "`" + ` FROM foo // -// The escaped string must be rendered inside an existing string literal +// SELECT ` + "`" + `group` + "`" + ` FROM foo +// +// # The escaped string must be rendered inside an existing string literal // // A string cannot be escaped twice func EscapeBacktick(s string) string { diff --git a/internal/engine/postgresql/convert.go b/internal/engine/postgresql/convert.go index 38a5e62a26..52d0d9546d 100644 --- a/internal/engine/postgresql/convert.go +++ b/internal/engine/postgresql/convert.go @@ -1,12 +1,9 @@ -//go:build !windows && cgo -// +build !windows,cgo - package postgresql import ( "fmt" - pg "github.com/pganalyze/pg_query_go/v4" + pg "github.com/wasilibs/go-pgquery" "github.com/sqlc-dev/sqlc/internal/sql/ast" ) diff --git a/internal/engine/postgresql/parse.go b/internal/engine/postgresql/parse.go index 296a14e858..37f29aa7fb 100644 --- a/internal/engine/postgresql/parse.go +++ b/internal/engine/postgresql/parse.go @@ -1,6 +1,3 @@ -//go:build !windows && cgo -// +build !windows,cgo - package postgresql import ( @@ -9,9 +6,9 @@ import ( "io" "strings" - nodes "github.com/pganalyze/pg_query_go/v4" - "github.com/pganalyze/pg_query_go/v4/parser" + nodes "github.com/wasilibs/go-pgquery" + "github.com/sqlc-dev/sqlc/internal/engine/postgresql/parser" "github.com/sqlc-dev/sqlc/internal/source" "github.com/sqlc-dev/sqlc/internal/sql/ast" "github.com/sqlc-dev/sqlc/internal/sql/sqlerr" @@ -154,7 +151,7 @@ func (p *Parser) Parse(r io.Reader) ([]ast.Statement, error) { if err != nil { return nil, err } - tree, err := nodes.Parse(string(contents)) + tree, err := parseNodes(string(contents)) if err != nil { pErr := normalizeErr(err) return nil, pErr diff --git a/internal/engine/postgresql/parse_default.go b/internal/engine/postgresql/parse_default.go new file mode 100644 index 0000000000..1dadcb5604 --- /dev/null +++ b/internal/engine/postgresql/parse_default.go @@ -0,0 +1,35 @@ +//go:build !windows && cgo +// +build !windows,cgo + +package postgresql + +import ( + cgonodes "github.com/pganalyze/pg_query_go/v4" + wasinodes "github.com/wasilibs/go-pgquery" + "google.golang.org/protobuf/proto" +) + +func parseNodes(input string) (*wasinodes.ParseResult, error) { + cgoRes, err := cgonodes.Parse(input) + if err != nil { + return nil, err + } + + // It would be too tedious to maintain conversion logic in a way that + // can target the two types from different packages despite being identical. + // We go ahead and take a small performance hit by marshaling through + // protobuf to unify the types. We must use the wasilibs version because + // the upstream version requires cgo even when only accessing the proto. + + resBytes, err := proto.Marshal(cgoRes) + if err != nil { + return nil, err + } + + var wasiRes wasinodes.ParseResult + if err := proto.Unmarshal(resBytes, &wasiRes); err != nil { + return nil, err + } + + return &wasiRes, nil +} diff --git a/internal/engine/postgresql/parse_disabled.go b/internal/engine/postgresql/parse_disabled.go deleted file mode 100644 index b07817b431..0000000000 --- a/internal/engine/postgresql/parse_disabled.go +++ /dev/null @@ -1,35 +0,0 @@ -//go:build windows || !cgo -// +build windows !cgo - -package postgresql - -import ( - "errors" - "io" - "runtime" - - "github.com/sqlc-dev/sqlc/internal/source" - "github.com/sqlc-dev/sqlc/internal/sql/ast" -) - -func NewParser() *Parser { - return &Parser{} -} - -type Parser struct { -} - -func (p *Parser) Parse(r io.Reader) ([]ast.Statement, error) { - if runtime.GOOS == "windows" { - return nil, errors.New("the PostgreSQL engine does not support Windows.") - } - return nil, errors.New("the PostgreSQL engine requires cgo. Please set CGO_ENABLED=1.") -} - -// https://www.postgresql.org/docs/current/sql-syntax-lexical.html#SQL-SYNTAX-COMMENTS -func (p *Parser) CommentSyntax() source.CommentSyntax { - return source.CommentSyntax{ - Dash: true, - SlashStar: true, - } -} diff --git a/internal/engine/postgresql/parse_wasi.go b/internal/engine/postgresql/parse_wasi.go new file mode 100644 index 0000000000..d0029368bc --- /dev/null +++ b/internal/engine/postgresql/parse_wasi.go @@ -0,0 +1,10 @@ +//go:build windows || !cgo +// +build windows !cgo + +package postgresql + +import ( + nodes "github.com/wasilibs/go-pgquery" +) + +var parseNodes = nodes.Parse diff --git a/internal/engine/postgresql/parser/parser_default.go b/internal/engine/postgresql/parser/parser_default.go new file mode 100644 index 0000000000..55555d1176 --- /dev/null +++ b/internal/engine/postgresql/parser/parser_default.go @@ -0,0 +1,8 @@ +//go:build !windows && cgo +// +build !windows,cgo + +package parser + +import "github.com/pganalyze/pg_query_go/v4/parser" + +type Error = parser.Error diff --git a/internal/engine/postgresql/parser/parser_wasi.go b/internal/engine/postgresql/parser/parser_wasi.go new file mode 100644 index 0000000000..d908b329a5 --- /dev/null +++ b/internal/engine/postgresql/parser/parser_wasi.go @@ -0,0 +1,8 @@ +//go:build windows || !cgo +// +build windows !cgo + +package parser + +import "github.com/wasilibs/go-pgquery/parser" + +type Error = parser.Error diff --git a/internal/engine/postgresql/utils.go b/internal/engine/postgresql/utils.go index d0ddc392a1..0ad2f45078 100644 --- a/internal/engine/postgresql/utils.go +++ b/internal/engine/postgresql/utils.go @@ -1,10 +1,7 @@ -//go:build !windows && cgo -// +build !windows,cgo - package postgresql import ( - nodes "github.com/pganalyze/pg_query_go/v4" + nodes "github.com/wasilibs/go-pgquery" ) func isArray(n *nodes.TypeName) bool { From f7c2c9c287d351b0bb9d0f2fa0abbc2c3b813388 Mon Sep 17 00:00:00 2001 From: Anuraag Agrawal Date: Fri, 1 Dec 2023 16:59:47 +0900 Subject: [PATCH 02/20] Fix --- .github/workflows/ci.yml | 27 +++++++++++++++++++++--- internal/endtoend/fmt_test.go | 7 +++--- internal/engine/postgresql/parse_wasi.go | 3 +++ 3 files changed, 30 insertions(+), 7 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 681733bae7..3ee34ce5d7 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -5,7 +5,7 @@ on: - main pull_request: jobs: - windows-build: + windows-test: if: ${{ github.ref == 'refs/heads/main' }} runs-on: windows-latest steps: @@ -14,9 +14,16 @@ jobs: with: go-version-file: go.mod check-latest: true + + - name: install sqlc-gen-test + run: go install github.com/sqlc-dev/sqlc-gen-test@v0.1.0 + + - name: install ./... + run: go install ./... + - run: go test ./... - darwin-build: + darwin-test: if: ${{ github.ref == 'refs/heads/main' }} runs-on: macos-latest steps: @@ -25,9 +32,16 @@ jobs: with: go-version-file: go.mod check-latest: true + + - name: install sqlc-gen-test + run: go install github.com/sqlc-dev/sqlc-gen-test@v0.1.0 + + - name: install ./... + run: go install ./... + - run: go test ./... - linux-purego-build: + linux-purego-test: if: ${{ github.ref == 'refs/heads/main' }} runs-on: macos-latest steps: @@ -36,6 +50,13 @@ jobs: with: go-version-file: go.mod check-latest: true + + - name: install sqlc-gen-test + run: go install github.com/sqlc-dev/sqlc-gen-test@v0.1.0 + + - name: install ./... + run: go install ./... + - run: go test ./... env: CGO_ENABLED: '0' diff --git a/internal/endtoend/fmt_test.go b/internal/endtoend/fmt_test.go index 22b5f1392d..04e753e5b7 100644 --- a/internal/endtoend/fmt_test.go +++ b/internal/endtoend/fmt_test.go @@ -8,7 +8,6 @@ import ( "strings" "testing" - pg_query "github.com/pganalyze/pg_query_go/v4" "github.com/sqlc-dev/sqlc/internal/debug" "github.com/sqlc-dev/sqlc/internal/engine/postgresql" "github.com/sqlc-dev/sqlc/internal/sql/ast" @@ -40,7 +39,7 @@ func TestFormat(t *testing.T) { } query := query t.Run(fmt.Sprintf("%d", i), func(t *testing.T) { - expected, err := pg_query.Fingerprint(string(query)) + expected, err := postgresql.Fingerprint(string(query)) if err != nil { t.Fatal(err) } @@ -52,12 +51,12 @@ func TestFormat(t *testing.T) { t.Fatal("expected one statement") } if false { - r, err := pg_query.Parse(string(query)) + r, err := postgresql.Parse(string(query)) debug.Dump(r, err) } out := ast.Format(stmts[0].Raw) - actual, err := pg_query.Fingerprint(out) + actual, err := postgresql.Fingerprint(out) if err != nil { t.Error(err) } diff --git a/internal/engine/postgresql/parse_wasi.go b/internal/engine/postgresql/parse_wasi.go index d0029368bc..27fcb9a4f1 100644 --- a/internal/engine/postgresql/parse_wasi.go +++ b/internal/engine/postgresql/parse_wasi.go @@ -8,3 +8,6 @@ import ( ) var parseNodes = nodes.Parse + +var Parse = parseNodes +var Fingerprint = cgonodes.Fingerprint From bae1e3cf904eb29958d114881b41635dc9a9e3a4 Mon Sep 17 00:00:00 2001 From: Anuraag Agrawal Date: Fri, 1 Dec 2023 17:00:14 +0900 Subject: [PATCH 03/20] Fix --- internal/engine/postgresql/parse_default.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/internal/engine/postgresql/parse_default.go b/internal/engine/postgresql/parse_default.go index 1dadcb5604..c458de3ad4 100644 --- a/internal/engine/postgresql/parse_default.go +++ b/internal/engine/postgresql/parse_default.go @@ -33,3 +33,6 @@ func parseNodes(input string) (*wasinodes.ParseResult, error) { return &wasiRes, nil } + +var Parse = parseNodes +var Fingerprint = cgonodes.Fingerprint From f223341c8305879ff8f452d885039e3f4518c4d6 Mon Sep 17 00:00:00 2001 From: Anuraag Agrawal Date: Fri, 1 Dec 2023 17:05:03 +0900 Subject: [PATCH 04/20] Typo --- internal/engine/postgresql/parse_wasi.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/engine/postgresql/parse_wasi.go b/internal/engine/postgresql/parse_wasi.go index 27fcb9a4f1..ae8cbdeff4 100644 --- a/internal/engine/postgresql/parse_wasi.go +++ b/internal/engine/postgresql/parse_wasi.go @@ -10,4 +10,4 @@ import ( var parseNodes = nodes.Parse var Parse = parseNodes -var Fingerprint = cgonodes.Fingerprint +var Fingerprint = nodes.Fingerprint From 4009fa4154037809e30439c4c01519026490acb3 Mon Sep 17 00:00:00 2001 From: Anuraag Agrawal Date: Fri, 1 Dec 2023 17:11:30 +0900 Subject: [PATCH 05/20] cgo guard for TestReplay --- internal/endtoend/endtoend_test.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/internal/endtoend/endtoend_test.go b/internal/endtoend/endtoend_test.go index aea72c5dbf..e5bd8b519e 100644 --- a/internal/endtoend/endtoend_test.go +++ b/internal/endtoend/endtoend_test.go @@ -1,3 +1,7 @@ +// Currently requires cgo for wasmtime. +//go:build cgo +// +build cgo + package main import ( From 315cb5a6e4396112bbcc918821dffc1370191c7c Mon Sep 17 00:00:00 2001 From: Anuraag Agrawal Date: Fri, 1 Dec 2023 17:15:36 +0900 Subject: [PATCH 06/20] Windows no test --- .github/workflows/ci.yml | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3ee34ce5d7..190ab7e61c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -5,7 +5,7 @@ on: - main pull_request: jobs: - windows-test: + windows-build: if: ${{ github.ref == 'refs/heads/main' }} runs-on: windows-latest steps: @@ -15,13 +15,9 @@ jobs: go-version-file: go.mod check-latest: true - - name: install sqlc-gen-test - run: go install github.com/sqlc-dev/sqlc-gen-test@v0.1.0 - - - name: install ./... - run: go install ./... - - - run: go test ./... + # Currently some tests rely on line endings so we only build + # without running. + - run: go build ./... darwin-test: if: ${{ github.ref == 'refs/heads/main' }} From 767e168b870fdcd94ff5a969cc9828165a6e84b0 Mon Sep 17 00:00:00 2001 From: Anuraag Agrawal Date: Sat, 2 Dec 2023 21:29:04 +0900 Subject: [PATCH 07/20] Use upstream nodes type --- go.mod | 4 +-- go.sum | 24 +++------------ internal/engine/postgresql/convert.go | 2 +- internal/engine/postgresql/parse.go | 4 +-- internal/engine/postgresql/parse_default.go | 33 ++------------------- internal/engine/postgresql/parse_wasi.go | 4 +-- internal/engine/postgresql/utils.go | 2 +- 7 files changed, 14 insertions(+), 59 deletions(-) diff --git a/go.mod b/go.mod index 384024a76e..5676c14e66 100644 --- a/go.mod +++ b/go.mod @@ -15,12 +15,12 @@ require ( github.com/jinzhu/inflection v1.0.0 github.com/lib/pq v1.10.9 github.com/mattn/go-sqlite3 v1.14.18 - github.com/pganalyze/pg_query_go/v4 v4.2.3 + github.com/pganalyze/pg_query_go/v4 v4.2.4-0.20231202022636-3c8cb1bd9d99 github.com/pingcap/tidb/pkg/parser v0.0.0-20231103154709-4f00ece106b1 github.com/riza-io/grpc-go v0.2.0 github.com/spf13/cobra v1.8.0 github.com/spf13/pflag v1.0.5 - github.com/wasilibs/go-pgquery v0.0.0-20231201074715-bbab8b1ab4a9 + github.com/wasilibs/go-pgquery v0.0.0-20231202122549-dbe58a6cd4c4 github.com/xeipuuv/gojsonschema v1.2.0 golang.org/x/sync v0.5.0 google.golang.org/grpc v1.59.0 diff --git a/go.sum b/go.sum index 106fde3f3f..86d69c9819 100644 --- a/go.sum +++ b/go.sum @@ -29,21 +29,11 @@ github.com/go-sql-driver/mysql v1.7.1/go.mod h1:OXbVy3sEdcQ2Doequ6Z5BW6fXNQTmx+9 github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/gofrs/uuid v4.0.0+incompatible h1:1SD/1F5pU8p29ybwgQSwpQk+mwdRrXCYuPhW6m+TnJw= github.com/gofrs/uuid v4.0.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= -github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= -github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= -github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= -github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= -github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= -github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/google/cel-go v0.18.2 h1:L0B6sNBSVmt0OyECi8v6VOS74KOc9W/tLiWKfZABvf4= github.com/google/cel-go v0.18.2/go.mod h1:kWcIzTsPX0zmQ+H3TirHstLLf9ep5QTsZBN9u4dOYLg= -github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= -github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= -github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= @@ -129,8 +119,8 @@ github.com/mattn/go-isatty v0.0.7/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hd github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= github.com/mattn/go-sqlite3 v1.14.18 h1:JL0eqdCOq6DJVNPSvArO/bIV9/P7fbGrV00LZHc+5aI= github.com/mattn/go-sqlite3 v1.14.18/go.mod h1:2eHXhiwb8IkHr+BDWZGa96P6+rkvnG63S2DGjv9HUNg= -github.com/pganalyze/pg_query_go/v4 v4.2.3 h1:cNLqyiVMasV7YGWyYV+fkXyHp32gDfXVNCqoHztEGNk= -github.com/pganalyze/pg_query_go/v4 v4.2.3/go.mod h1:aEkDNOXNM5j0YGzaAapwJ7LB3dLNj+bvbWcLv1hOVqA= +github.com/pganalyze/pg_query_go/v4 v4.2.4-0.20231202022636-3c8cb1bd9d99 h1:AarAXDJB+iIetitFJncVelAi7t/u+m3c+pkRNkEgTzg= +github.com/pganalyze/pg_query_go/v4 v4.2.4-0.20231202022636-3c8cb1bd9d99/go.mod h1:pRJ9rCcWIhDLAzymhH2iY5kEVEfU6UGrL1wBWuO7gmM= github.com/pingcap/errors v0.11.0/go.mod h1:Oi8TUi2kEtXXLMJk9l1cGmz20kV3TaQ0usTwv5KuLY8= github.com/pingcap/errors v0.11.4/go.mod h1:Oi8TUi2kEtXXLMJk9l1cGmz20kV3TaQ0usTwv5KuLY8= github.com/pingcap/errors v0.11.5-0.20210425183316-da1aaba5fb63 h1:+FZIDR/D97YOPik4N4lPDaUcLDF/EQPogxtlHB2ZZRM= @@ -185,8 +175,8 @@ github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= -github.com/wasilibs/go-pgquery v0.0.0-20231201074715-bbab8b1ab4a9 h1:JrDBgtPAsn6LC0EMn9jBaSFiZqVpKMa9hea47/nkaeA= -github.com/wasilibs/go-pgquery v0.0.0-20231201074715-bbab8b1ab4a9/go.mod h1:kalxVxSjXzPXnzVrIsejhYyPa81QLKHbNnNSB8bL6q0= +github.com/wasilibs/go-pgquery v0.0.0-20231202122549-dbe58a6cd4c4 h1:l8cvYIpTBSHZtRKk6nDZzP7j7GdM4OeSZFy+eSLdLUA= +github.com/wasilibs/go-pgquery v0.0.0-20231202122549-dbe58a6cd4c4/go.mod h1:BxeTakf417NHQfvSGi4BAzEeTFw8A2dIGXfECy4CyJU= github.com/wasilibs/wazerox v0.0.0-20231117065139-b3503f4aeff6 h1:jwbU8u5TuXModzdEG4wI0g4FyuD7ROSttU86go5sPdU= github.com/wasilibs/wazerox v0.0.0-20231117065139-b3503f4aeff6/go.mod h1:IQNVyA4d1hWIe23mlMMuqXjyWMdndgSlNx6FqBkwPsM= github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f h1:J9EGpcZtP0E/raorCMxlFGSTBrsSlaDGf3jU/qvAE2c= @@ -307,12 +297,6 @@ google.golang.org/genproto/googleapis/rpc v0.0.0-20230822172742-b8732ec3820d h1: google.golang.org/genproto/googleapis/rpc v0.0.0-20230822172742-b8732ec3820d/go.mod h1:+Bk1OCOj40wS2hwAMA+aCW9ypzm63QTBBHp6lQ3p+9M= google.golang.org/grpc v1.59.0 h1:Z5Iec2pjwb+LEOqzpB2MR12/eKFhDPhuqW91O+4bwUk= google.golang.org/grpc v1.59.0/go.mod h1:aUPDwccQo6OTjy7Hct4AfBPD1GptF4fyUjIkQ9YtF98= -google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= -google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= -google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= -google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= -google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= -google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8= diff --git a/internal/engine/postgresql/convert.go b/internal/engine/postgresql/convert.go index 52d0d9546d..82bd1fb7db 100644 --- a/internal/engine/postgresql/convert.go +++ b/internal/engine/postgresql/convert.go @@ -3,7 +3,7 @@ package postgresql import ( "fmt" - pg "github.com/wasilibs/go-pgquery" + pg "github.com/pganalyze/pg_query_go/v4" "github.com/sqlc-dev/sqlc/internal/sql/ast" ) diff --git a/internal/engine/postgresql/parse.go b/internal/engine/postgresql/parse.go index 37f29aa7fb..d3c8bae217 100644 --- a/internal/engine/postgresql/parse.go +++ b/internal/engine/postgresql/parse.go @@ -6,7 +6,7 @@ import ( "io" "strings" - nodes "github.com/wasilibs/go-pgquery" + nodes "github.com/pganalyze/pg_query_go/v4" "github.com/sqlc-dev/sqlc/internal/engine/postgresql/parser" "github.com/sqlc-dev/sqlc/internal/source" @@ -151,7 +151,7 @@ func (p *Parser) Parse(r io.Reader) ([]ast.Statement, error) { if err != nil { return nil, err } - tree, err := parseNodes(string(contents)) + tree, err := Parse(string(contents)) if err != nil { pErr := normalizeErr(err) return nil, pErr diff --git a/internal/engine/postgresql/parse_default.go b/internal/engine/postgresql/parse_default.go index c458de3ad4..993ac607e2 100644 --- a/internal/engine/postgresql/parse_default.go +++ b/internal/engine/postgresql/parse_default.go @@ -4,35 +4,8 @@ package postgresql import ( - cgonodes "github.com/pganalyze/pg_query_go/v4" - wasinodes "github.com/wasilibs/go-pgquery" - "google.golang.org/protobuf/proto" + nodes "github.com/pganalyze/pg_query_go/v4" ) -func parseNodes(input string) (*wasinodes.ParseResult, error) { - cgoRes, err := cgonodes.Parse(input) - if err != nil { - return nil, err - } - - // It would be too tedious to maintain conversion logic in a way that - // can target the two types from different packages despite being identical. - // We go ahead and take a small performance hit by marshaling through - // protobuf to unify the types. We must use the wasilibs version because - // the upstream version requires cgo even when only accessing the proto. - - resBytes, err := proto.Marshal(cgoRes) - if err != nil { - return nil, err - } - - var wasiRes wasinodes.ParseResult - if err := proto.Unmarshal(resBytes, &wasiRes); err != nil { - return nil, err - } - - return &wasiRes, nil -} - -var Parse = parseNodes -var Fingerprint = cgonodes.Fingerprint +var Parse = nodes.Parse +var Fingerprint = nodes.Fingerprint diff --git a/internal/engine/postgresql/parse_wasi.go b/internal/engine/postgresql/parse_wasi.go index ae8cbdeff4..0cd4033082 100644 --- a/internal/engine/postgresql/parse_wasi.go +++ b/internal/engine/postgresql/parse_wasi.go @@ -7,7 +7,5 @@ import ( nodes "github.com/wasilibs/go-pgquery" ) -var parseNodes = nodes.Parse - -var Parse = parseNodes +var Parse = nodes.Parse var Fingerprint = nodes.Fingerprint diff --git a/internal/engine/postgresql/utils.go b/internal/engine/postgresql/utils.go index 0ad2f45078..15b1c49d0c 100644 --- a/internal/engine/postgresql/utils.go +++ b/internal/engine/postgresql/utils.go @@ -1,7 +1,7 @@ package postgresql import ( - nodes "github.com/wasilibs/go-pgquery" + nodes "github.com/pganalyze/pg_query_go/v4" ) func isArray(n *nodes.TypeName) bool { From a608c498b63b4ec09a94a61b134ab1595f7dc6f9 Mon Sep 17 00:00:00 2001 From: Anuraag Agrawal Date: Tue, 5 Dec 2023 13:31:41 +0900 Subject: [PATCH 08/20] Update to latest pg_query_go --- go.mod | 4 ++-- go.sum | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/go.mod b/go.mod index 5676c14e66..530c9791bd 100644 --- a/go.mod +++ b/go.mod @@ -15,12 +15,12 @@ require ( github.com/jinzhu/inflection v1.0.0 github.com/lib/pq v1.10.9 github.com/mattn/go-sqlite3 v1.14.18 - github.com/pganalyze/pg_query_go/v4 v4.2.4-0.20231202022636-3c8cb1bd9d99 + github.com/pganalyze/pg_query_go/v4 v4.2.4-0.20231205012101-7463430c7b73 github.com/pingcap/tidb/pkg/parser v0.0.0-20231103154709-4f00ece106b1 github.com/riza-io/grpc-go v0.2.0 github.com/spf13/cobra v1.8.0 github.com/spf13/pflag v1.0.5 - github.com/wasilibs/go-pgquery v0.0.0-20231202122549-dbe58a6cd4c4 + github.com/wasilibs/go-pgquery v0.0.0-20231205013331-96e794bb074e github.com/xeipuuv/gojsonschema v1.2.0 golang.org/x/sync v0.5.0 google.golang.org/grpc v1.59.0 diff --git a/go.sum b/go.sum index 86d69c9819..112bcd5436 100644 --- a/go.sum +++ b/go.sum @@ -119,8 +119,8 @@ github.com/mattn/go-isatty v0.0.7/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hd github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= github.com/mattn/go-sqlite3 v1.14.18 h1:JL0eqdCOq6DJVNPSvArO/bIV9/P7fbGrV00LZHc+5aI= github.com/mattn/go-sqlite3 v1.14.18/go.mod h1:2eHXhiwb8IkHr+BDWZGa96P6+rkvnG63S2DGjv9HUNg= -github.com/pganalyze/pg_query_go/v4 v4.2.4-0.20231202022636-3c8cb1bd9d99 h1:AarAXDJB+iIetitFJncVelAi7t/u+m3c+pkRNkEgTzg= -github.com/pganalyze/pg_query_go/v4 v4.2.4-0.20231202022636-3c8cb1bd9d99/go.mod h1:pRJ9rCcWIhDLAzymhH2iY5kEVEfU6UGrL1wBWuO7gmM= +github.com/pganalyze/pg_query_go/v4 v4.2.4-0.20231205012101-7463430c7b73 h1:vZEujakYrBzd7CWMwqTsDRU7bdRm4Aiu76X9K3P+qtI= +github.com/pganalyze/pg_query_go/v4 v4.2.4-0.20231205012101-7463430c7b73/go.mod h1:pRJ9rCcWIhDLAzymhH2iY5kEVEfU6UGrL1wBWuO7gmM= github.com/pingcap/errors v0.11.0/go.mod h1:Oi8TUi2kEtXXLMJk9l1cGmz20kV3TaQ0usTwv5KuLY8= github.com/pingcap/errors v0.11.4/go.mod h1:Oi8TUi2kEtXXLMJk9l1cGmz20kV3TaQ0usTwv5KuLY8= github.com/pingcap/errors v0.11.5-0.20210425183316-da1aaba5fb63 h1:+FZIDR/D97YOPik4N4lPDaUcLDF/EQPogxtlHB2ZZRM= @@ -175,8 +175,8 @@ github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= -github.com/wasilibs/go-pgquery v0.0.0-20231202122549-dbe58a6cd4c4 h1:l8cvYIpTBSHZtRKk6nDZzP7j7GdM4OeSZFy+eSLdLUA= -github.com/wasilibs/go-pgquery v0.0.0-20231202122549-dbe58a6cd4c4/go.mod h1:BxeTakf417NHQfvSGi4BAzEeTFw8A2dIGXfECy4CyJU= +github.com/wasilibs/go-pgquery v0.0.0-20231205013331-96e794bb074e h1:sGIC6/D0KqpA+qBSDSVDQswU/IJVYkbnUXnipgTLQWk= +github.com/wasilibs/go-pgquery v0.0.0-20231205013331-96e794bb074e/go.mod h1:KW0azBSWqkPZ71r+3O4qt8h6A/NisFLp0rbjZ3py4OE= github.com/wasilibs/wazerox v0.0.0-20231117065139-b3503f4aeff6 h1:jwbU8u5TuXModzdEG4wI0g4FyuD7ROSttU86go5sPdU= github.com/wasilibs/wazerox v0.0.0-20231117065139-b3503f4aeff6/go.mod h1:IQNVyA4d1hWIe23mlMMuqXjyWMdndgSlNx6FqBkwPsM= github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f h1:J9EGpcZtP0E/raorCMxlFGSTBrsSlaDGf3jU/qvAE2c= From ce0fa99dcf610106a6e2964300c322b4855dbb93 Mon Sep 17 00:00:00 2001 From: Anuraag Agrawal Date: Tue, 5 Dec 2023 13:43:41 +0900 Subject: [PATCH 09/20] Test matrix --- .github/workflows/ci.yml | 76 +++++++++++----------------------------- 1 file changed, 21 insertions(+), 55 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 190ab7e61c..2f27b5579c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -5,61 +5,22 @@ on: - main pull_request: jobs: - windows-build: - if: ${{ github.ref == 'refs/heads/main' }} - runs-on: windows-latest - steps: - - uses: actions/checkout@v4 - - uses: actions/setup-go@v4 - with: - go-version-file: go.mod - check-latest: true - - # Currently some tests rely on line endings so we only build - # without running. - - run: go build ./... - - darwin-test: - if: ${{ github.ref == 'refs/heads/main' }} - runs-on: macos-latest - steps: - - uses: actions/checkout@v4 - - uses: actions/setup-go@v4 - with: - go-version-file: go.mod - check-latest: true - - - name: install sqlc-gen-test - run: go install github.com/sqlc-dev/sqlc-gen-test@v0.1.0 - - - name: install ./... - run: go install ./... - - - run: go test ./... - - linux-purego-test: - if: ${{ github.ref == 'refs/heads/main' }} - runs-on: macos-latest - steps: - - uses: actions/checkout@v4 - - uses: actions/setup-go@v4 - with: - go-version-file: go.mod - check-latest: true - - - name: install sqlc-gen-test - run: go install github.com/sqlc-dev/sqlc-gen-test@v0.1.0 - - - name: install ./... - run: go install ./... - - - run: go test ./... - env: - CGO_ENABLED: '0' - - build: - name: test - runs-on: ubuntu-latest + test: + strategy: + max-parallel: 2 + matrix: + os: [windows-latest, macos-latest, ubuntu-latest] + cgo: ['1', '0'] + # Workaround no proper support for conditional matrix items + # https://github.com/orgs/community/discussions/26253#discussioncomment-6745038 + isMain: ${{ github.ref == 'refs/heads/main' }} + exclude: + - { isMain: false } + include: + - { os: ubuntu-latest, cgo: '1' } + - { os: ubuntu-latest, cgo: '0' } + name: Test ${{ matrix.os }}-cgo${{ matrix.cgo }} + runs-on: ${{ matrix.os }} steps: - uses: actions/checkout@v4 @@ -76,10 +37,14 @@ jobs: - name: install ./... run: go install ./... + env: + CGO_ENABLED: ${{ matrix.cgo }} - name: build internal/endtoend run: go build ./... working-directory: internal/endtoend/testdata + env: + CGO_ENABLED: ${{ matrix.cgo }} - name: test ./... run: gotestsum --junitfile junit.xml -- --tags=examples -timeout 20m ./... @@ -87,6 +52,7 @@ jobs: CI_SQLC_PROJECT_ID: ${{ secrets.CI_SQLC_PROJECT_ID }} CI_SQLC_AUTH_TOKEN: ${{ secrets.CI_SQLC_AUTH_TOKEN }} SQLC_AUTH_TOKEN: ${{ secrets.CI_SQLC_AUTH_TOKEN }} + CGO_ENABLED: ${{ matrix.cgo }} vuln_check: runs-on: ubuntu-latest From 61e9a97fbeb395b57e0fe17127af871ce6c95797 Mon Sep 17 00:00:00 2001 From: Anuraag Agrawal Date: Tue, 5 Dec 2023 13:50:09 +0900 Subject: [PATCH 10/20] Tweak workflow --- .github/workflows/ci.yml | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2f27b5579c..5d42f64869 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -15,10 +15,12 @@ jobs: # https://github.com/orgs/community/discussions/26253#discussioncomment-6745038 isMain: ${{ github.ref == 'refs/heads/main' }} exclude: - - { isMain: false } + - isMain: false include: - - { os: ubuntu-latest, cgo: '1' } - - { os: ubuntu-latest, cgo: '0' } + - os: ubuntu-latest + cgo: '1' + - os: ubuntu-latest + cgo: '0' name: Test ${{ matrix.os }}-cgo${{ matrix.cgo }} runs-on: ${{ matrix.os }} From d12d5747c39db43facb5c58d0500c6a3fe369364 Mon Sep 17 00:00:00 2001 From: Anuraag Agrawal Date: Tue, 5 Dec 2023 13:51:26 +0900 Subject: [PATCH 11/20] Fix workflow --- .github/workflows/ci.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5d42f64869..6771315109 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -13,7 +13,8 @@ jobs: cgo: ['1', '0'] # Workaround no proper support for conditional matrix items # https://github.com/orgs/community/discussions/26253#discussioncomment-6745038 - isMain: ${{ github.ref == 'refs/heads/main' }} + isMain: + - ${{ github.ref == 'refs/heads/main' }} exclude: - isMain: false include: From e0af7a935755991722f75ef2b534fa9e9803fc74 Mon Sep 17 00:00:00 2001 From: Anuraag Agrawal Date: Tue, 5 Dec 2023 13:53:38 +0900 Subject: [PATCH 12/20] Tweak job name --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6771315109..ea4c2af715 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -22,7 +22,7 @@ jobs: cgo: '1' - os: ubuntu-latest cgo: '0' - name: Test ${{ matrix.os }}-cgo${{ matrix.cgo }} + name: test ${{ matrix.os }} cgo=${{ matrix.cgo }} runs-on: ${{ matrix.os }} steps: From 2d8bb6913a2a18e4f506c6e8d70b9922b3696cb4 Mon Sep 17 00:00:00 2001 From: Anuraag Agrawal Date: Tue, 5 Dec 2023 14:07:44 +0900 Subject: [PATCH 13/20] Fix tests for nocgo / windows --- examples/authors/sqlite/db_test.go | 4 ++-- examples/booktest/sqlite/db_test.go | 4 ++-- examples/ondeck/sqlite/db_test.go | 4 ++-- internal/sql/sqlpath/read_test.go | 34 +++++++++++++++-------------- 4 files changed, 24 insertions(+), 22 deletions(-) diff --git a/examples/authors/sqlite/db_test.go b/examples/authors/sqlite/db_test.go index b07804b627..d47430f4cf 100644 --- a/examples/authors/sqlite/db_test.go +++ b/examples/authors/sqlite/db_test.go @@ -1,5 +1,5 @@ -//go:build examples -// +build examples +//go:build examples && cgo +// +build examples,cgo package authors diff --git a/examples/booktest/sqlite/db_test.go b/examples/booktest/sqlite/db_test.go index f169c79148..7580940888 100644 --- a/examples/booktest/sqlite/db_test.go +++ b/examples/booktest/sqlite/db_test.go @@ -1,5 +1,5 @@ -//go:build examples -// +build examples +//go:build examples && cgo +// +build examples,cgo package booktest diff --git a/examples/ondeck/sqlite/db_test.go b/examples/ondeck/sqlite/db_test.go index f106256d45..694a8d5b58 100644 --- a/examples/ondeck/sqlite/db_test.go +++ b/examples/ondeck/sqlite/db_test.go @@ -1,5 +1,5 @@ -//go:build examples -// +build examples +//go:build examples && cgo +// +build examples,cgo package ondeck diff --git a/internal/sql/sqlpath/read_test.go b/internal/sql/sqlpath/read_test.go index babb80cc80..ddf4d0909a 100644 --- a/internal/sql/sqlpath/read_test.go +++ b/internal/sql/sqlpath/read_test.go @@ -2,6 +2,8 @@ package sqlpath import ( "fmt" + "path/filepath" + "strings" "testing" "github.com/google/go-cmp/cmp" @@ -107,8 +109,8 @@ func TestReturnsErrorWhenPathDoesNotExist(t *testing.T) { if err == nil { t.Errorf("Expected an error, but got nil") } else { - expectedError := fmt.Errorf("path error: stat non_existent_path: no such file or directory") - if !cmp.Equal(err.Error(), expectedError.Error()) { + expectedError := fmt.Errorf("path error:") + if !strings.HasPrefix(err.Error(), expectedError.Error()) { t.Errorf("Expected error %v, but got %v", expectedError, err) } } @@ -129,8 +131,8 @@ func TestReturnsErrorWhenDirectoryCannotBeRead(t *testing.T) { if err == nil { t.Errorf("Expected an error, but got nil") } else { - expectedError := fmt.Errorf("path error: stat testdata/unreadable: no such file or directory") - if !cmp.Equal(err.Error(), expectedError.Error()) { + expectedError := fmt.Errorf("path error:") + if !strings.HasPrefix(err.Error(), expectedError.Error()) { t.Errorf("Expected error %v, but got %v", expectedError, err) } } @@ -194,10 +196,10 @@ func TestFollowSymlinks(t *testing.T) { // Assert expected := []string{ - "testdata/symlink/file1.sql", - "testdata/symlink/file1.symlink.sql", - "testdata/symlink/file2.sql", - "testdata/file1.symlink.sql", + filepath.Join("testdata", "symlink", "file1.sql"), + filepath.Join("testdata", "symlink", "file1.symlink.sql"), + filepath.Join("testdata", "symlink", "file2.sql"), + filepath.Join("testdata", "file1.symlink.sql"), } if !cmp.Equal(result, expected) { t.Errorf("Expected %v, but got %v", expected, result) @@ -216,24 +218,24 @@ func TestGlobPattern(t *testing.T) { { pattern: "testdata/glob/*/queries", expected: []string{ - "testdata/glob/sub1/queries/file1.sql", - "testdata/glob/sub2/queries/file2.sql", - "testdata/glob/sub3/queries/file3.sql", - "testdata/glob/sub3/queries/file4.sql", + filepath.Join("testdata", "glob", "sub1", "queries", "file1.sql"), + filepath.Join("testdata", "glob", "sub2", "queries", "file2.sql"), + filepath.Join("testdata", "glob", "sub3", "queries", "file3.sql"), + filepath.Join("testdata", "glob", "sub3", "queries", "file4.sql"), }, }, { pattern: "testdata/glob/sub3/queries/file?.sql", expected: []string{ - "testdata/glob/sub3/queries/file3.sql", - "testdata/glob/sub3/queries/file4.sql", + filepath.Join("testdata", "glob", "sub3", "queries", "file3.sql"), + filepath.Join("testdata", "glob", "sub3", "queries", "file4.sql"), }, }, { pattern: "testdata/glob/sub3/queries/file[1-5].sql", expected: []string{ - "testdata/glob/sub3/queries/file3.sql", - "testdata/glob/sub3/queries/file4.sql", + filepath.Join("testdata", "glob", "sub3", "queries", "file3.sql"), + filepath.Join("testdata", "glob", "sub3", "queries", "file4.sql"), }, }, } From 80881e71a1dda5825aa92d93090deb84da15573c Mon Sep 17 00:00:00 2001 From: Anuraag Agrawal Date: Tue, 5 Dec 2023 14:09:07 +0900 Subject: [PATCH 14/20] Parallel --- .github/workflows/ci.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ea4c2af715..2f80b93f8f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -7,7 +7,6 @@ on: jobs: test: strategy: - max-parallel: 2 matrix: os: [windows-latest, macos-latest, ubuntu-latest] cgo: ['1', '0'] From 03709bb953291854fae9532a9b4cbe716a9aa90d Mon Sep 17 00:00:00 2001 From: Anuraag Agrawal Date: Tue, 5 Dec 2023 14:22:19 +0900 Subject: [PATCH 15/20] Clean --- internal/sql/sqlpath/read.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/sql/sqlpath/read.go b/internal/sql/sqlpath/read.go index 02c8b2855c..278ebf3a05 100644 --- a/internal/sql/sqlpath/read.go +++ b/internal/sql/sqlpath/read.go @@ -46,7 +46,7 @@ func Glob(patterns []string) ([]string, error) { files = append(files, filepath.Join(path, f.Name())) } } else { - files = append(files, path) + files = append(files, filepath.Clean(path)) } } var sqlFiles []string From 918bcbae22d930aa275e2c5964eaf2d18f36423c Mon Sep 17 00:00:00 2001 From: Anuraag Agrawal Date: Tue, 5 Dec 2023 14:31:22 +0900 Subject: [PATCH 16/20] Fix test --- .github/workflows/ci.yml | 2 +- internal/sql/sqlpath/read_test.go | 22 +++++----------------- 2 files changed, 6 insertions(+), 18 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2f80b93f8f..24c9844b60 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -8,7 +8,7 @@ jobs: test: strategy: matrix: - os: [windows-latest, macos-latest, ubuntu-latest] + os: [ubuntu-latest, macos-latest, windows-latest] cgo: ['1', '0'] # Workaround no proper support for conditional matrix items # https://github.com/orgs/community/discussions/26253#discussioncomment-6745038 diff --git a/internal/sql/sqlpath/read_test.go b/internal/sql/sqlpath/read_test.go index ddf4d0909a..e4d959085b 100644 --- a/internal/sql/sqlpath/read_test.go +++ b/internal/sql/sqlpath/read_test.go @@ -18,7 +18,7 @@ func TestReturnsListOfSQLFiles(t *testing.T) { result, err := Glob(paths) // Assert - expected := []string{"testdata/file1.sql", "testdata/file2.sql"} + expected := []string{filepath.Join("testdata", "file1.sql"), filepath.Join("testdata", "file2.sql")} if !cmp.Equal(result, expected) { t.Errorf("Expected %v, but got %v, %v", expected, result, cmp.Diff(expected, result)) } @@ -85,7 +85,7 @@ func TestExcludesSQLFilesEndingWithDownSQLWhenSearchingForSQLFiles(t *testing.T) result, err := Glob(paths) // Assert - expected := []string{"testdata/file1.sql"} + expected := []string{filepath.Join("testdata", "file1.sql")} if !cmp.Equal(result, expected) { t.Errorf("Expected %v, but got %v", expected, result) } @@ -166,22 +166,10 @@ func TestNotIncludesHiddenFilesAnyPath(t *testing.T) { result, err := Glob(paths) // Assert - expectedAny := [][]string{ - {"./testdata/.hiddendir/file1.sql"}, - {"testdata/.hiddendir/file1.sql"}, - } - - match := false - for _, expected := range expectedAny { - if cmp.Equal(result, expected) { - match = true - break - } - } - if !match { - t.Errorf("Expected any of %v, but got %v", expectedAny, result) + expected := []string{filepath.Join("testdata", ".hiddendir", "file1.sql")} + if !cmp.Equal(result, expected) { + t.Errorf("Expected %v, but got %v", expected, result) } - if err != nil { t.Errorf("Expected no error, but got %v", err) } From 720d2f9e5870e7479c1462c3685f1eb6c51d1236 Mon Sep 17 00:00:00 2001 From: Anuraag Agrawal Date: Tue, 5 Dec 2023 14:43:03 +0900 Subject: [PATCH 17/20] Use windows line ending in test --- .github/workflows/ci.yml | 2 +- internal/endtoend/endtoend_test.go | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 24c9844b60..647bc88c34 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -10,7 +10,7 @@ jobs: matrix: os: [ubuntu-latest, macos-latest, windows-latest] cgo: ['1', '0'] - # Workaround no proper support for conditional matrix items + # Workaround no native support for conditional matrix items # https://github.com/orgs/community/discussions/26253#discussioncomment-6745038 isMain: - ${{ github.ref == 'refs/heads/main' }} diff --git a/internal/endtoend/endtoend_test.go b/internal/endtoend/endtoend_test.go index e5bd8b519e..964ad62b25 100644 --- a/internal/endtoend/endtoend_test.go +++ b/internal/endtoend/endtoend_test.go @@ -10,6 +10,7 @@ import ( "os" osexec "os/exec" "path/filepath" + "runtime" "slices" "strings" "testing" @@ -245,6 +246,9 @@ func cmpDirectory(t *testing.T, dir string, actual map[string]string) { t.Errorf("%s is empty", name) return } + if runtime.GOOS == "windows" { + contents = strings.ReplaceAll(contents, "\n", "\r\n") + } if diff := cmp.Diff(contents, actual[name]); diff != "" { t.Errorf("%s differed (-want +got):\n%s", name, diff) } From c2d0f1decae622e51638ded20e1482b6e495124f Mon Sep 17 00:00:00 2001 From: Anuraag Agrawal Date: Tue, 5 Dec 2023 14:55:12 +0900 Subject: [PATCH 18/20] Fix --- internal/endtoend/endtoend_test.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/internal/endtoend/endtoend_test.go b/internal/endtoend/endtoend_test.go index 964ad62b25..5ee0420970 100644 --- a/internal/endtoend/endtoend_test.go +++ b/internal/endtoend/endtoend_test.go @@ -247,7 +247,9 @@ func cmpDirectory(t *testing.T, dir string, actual map[string]string) { return } if runtime.GOOS == "windows" { - contents = strings.ReplaceAll(contents, "\n", "\r\n") + // TODO: It may be better to use .gitattributes to make sure the testdata + // has correct line endings on windows too. + contents = strings.ReplaceAll(contents, "\r\n", "\n") } if diff := cmp.Diff(contents, actual[name]); diff != "" { t.Errorf("%s differed (-want +got):\n%s", name, diff) From 64078d777184f8a605f861ee616b97aaf2955b37 Mon Sep 17 00:00:00 2001 From: Anuraag Agrawal Date: Tue, 5 Dec 2023 15:07:16 +0900 Subject: [PATCH 19/20] Try more --- internal/endtoend/endtoend_test.go | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/internal/endtoend/endtoend_test.go b/internal/endtoend/endtoend_test.go index 5ee0420970..98a3519511 100644 --- a/internal/endtoend/endtoend_test.go +++ b/internal/endtoend/endtoend_test.go @@ -144,6 +144,11 @@ func TestReplay(t *testing.T) { args = &Exec{Command: "generate"} } expected := string(tc.Stderr) + if runtime.GOOS == "windows" { + // TODO: It may be better to use .gitattributes to make sure the testdata + // has correct line endings on windows too. + expected = strings.ReplaceAll(expected, "\r\n", "\n") + } if args.Process != "" { _, err := osexec.LookPath(args.Process) @@ -231,7 +236,13 @@ func cmpDirectory(t *testing.T, dir string, actual map[string]string) { if err != nil { return err } - expected[path] = string(blob) + contents := string(blob) + if runtime.GOOS == "windows" { + // TODO: It may be better to use .gitattributes to make sure the testdata + // has correct line endings on windows too. + contents = strings.ReplaceAll(contents, "\r\n", "\n") + } + expected[path] = contents return nil } if err := filepath.Walk(dir, ff); err != nil { @@ -246,11 +257,6 @@ func cmpDirectory(t *testing.T, dir string, actual map[string]string) { t.Errorf("%s is empty", name) return } - if runtime.GOOS == "windows" { - // TODO: It may be better to use .gitattributes to make sure the testdata - // has correct line endings on windows too. - contents = strings.ReplaceAll(contents, "\r\n", "\n") - } if diff := cmp.Diff(contents, actual[name]); diff != "" { t.Errorf("%s differed (-want +got):\n%s", name, diff) } From 1ee327218e248138c5ddda2133138cfe53bfa7b3 Mon Sep 17 00:00:00 2001 From: Anuraag Agrawal Date: Tue, 5 Dec 2023 15:17:47 +0900 Subject: [PATCH 20/20] Ignore e2e on win --- internal/endtoend/endtoend_test.go | 20 ++++---------------- 1 file changed, 4 insertions(+), 16 deletions(-) diff --git a/internal/endtoend/endtoend_test.go b/internal/endtoend/endtoend_test.go index 98a3519511..10cd065c37 100644 --- a/internal/endtoend/endtoend_test.go +++ b/internal/endtoend/endtoend_test.go @@ -1,6 +1,6 @@ -// Currently requires cgo for wasmtime. -//go:build cgo -// +build cgo +// Currently requires cgo for wasmtime and has line-ending issues on windows. +//go:build cgo && !windows +// +build cgo,!windows package main @@ -10,7 +10,6 @@ import ( "os" osexec "os/exec" "path/filepath" - "runtime" "slices" "strings" "testing" @@ -144,11 +143,6 @@ func TestReplay(t *testing.T) { args = &Exec{Command: "generate"} } expected := string(tc.Stderr) - if runtime.GOOS == "windows" { - // TODO: It may be better to use .gitattributes to make sure the testdata - // has correct line endings on windows too. - expected = strings.ReplaceAll(expected, "\r\n", "\n") - } if args.Process != "" { _, err := osexec.LookPath(args.Process) @@ -236,13 +230,7 @@ func cmpDirectory(t *testing.T, dir string, actual map[string]string) { if err != nil { return err } - contents := string(blob) - if runtime.GOOS == "windows" { - // TODO: It may be better to use .gitattributes to make sure the testdata - // has correct line endings on windows too. - contents = strings.ReplaceAll(contents, "\r\n", "\n") - } - expected[path] = contents + expected[path] = string(blob) return nil } if err := filepath.Walk(dir, ff); err != nil {