From 75ab062f2c5e14476798d24482e1ef078cd6e0c5 Mon Sep 17 00:00:00 2001 From: Terri Cain Date: Fri, 14 Jun 2024 21:55:37 +0100 Subject: [PATCH 1/4] Updated MySQL copyfrom template to handle strucs with 1 argument --- .../golang/templates/go-sql-driver-mysql/copyfromCopy.tmpl | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/internal/codegen/golang/templates/go-sql-driver-mysql/copyfromCopy.tmpl b/internal/codegen/golang/templates/go-sql-driver-mysql/copyfromCopy.tmpl index e6b9061d31..e21475b148 100644 --- a/internal/codegen/golang/templates/go-sql-driver-mysql/copyfromCopy.tmpl +++ b/internal/codegen/golang/templates/go-sql-driver-mysql/copyfromCopy.tmpl @@ -9,11 +9,11 @@ func convertRowsFor{{.MethodName}}(w *io.PipeWriter, {{.Arg.SlicePair}}) { {{- with $arg := .Arg }} {{- range $arg.CopyFromMySQLFields}} {{- if eq .Type "string"}} - e.AppendString({{if eq (len $arg.CopyFromMySQLFields) 1}}row{{else}}row.{{.Name}}{{end}}) + e.AppendString({{if $arg.Struct}}row.{{.Name}}{{else}}row{{end}}) {{- else if or (eq .Type "[]byte") (eq .Type "json.RawMessage")}} - e.AppendBytes({{if eq (len $arg.CopyFromMySQLFields) 1}}row{{else}}row.{{.Name}}{{end}}) + e.AppendBytes({{if $arg.Struct}}row.{{.Name}}{{else}}row{{end}}) {{- else}} - e.AppendValue({{if eq (len $arg.CopyFromMySQLFields) 1}}row{{else}}row.{{.Name}}{{end}}) + e.AppendValue({{if $arg.Struct}}row.{{.Name}}{{else}}row{{end}}) {{- end}} {{- end}} {{- end}} From 654e4a93f36b223c55ae29b808dbee1c7aedec85 Mon Sep 17 00:00:00 2001 From: Terri Cain Date: Fri, 14 Jun 2024 21:55:59 +0100 Subject: [PATCH 2/4] Added copyfrom 1 arg struct testcase --- .../issue.md | 1 + .../mysql/go/copyfrom.go | 50 +++++++++++++++++++ .../mysql/go/db.go | 31 ++++++++++++ .../mysql/go/models.go | 13 +++++ .../mysql/go/query.sql.go | 18 +++++++ .../mysql/query.sql | 2 + .../mysql/schema.sql | 1 + .../mysql/sqlc.json | 15 ++++++ 8 files changed, 131 insertions(+) create mode 100644 internal/endtoend/testdata/copyfrom_singlecolumn_struct_only/issue.md create mode 100644 internal/endtoend/testdata/copyfrom_singlecolumn_struct_only/mysql/go/copyfrom.go create mode 100644 internal/endtoend/testdata/copyfrom_singlecolumn_struct_only/mysql/go/db.go create mode 100644 internal/endtoend/testdata/copyfrom_singlecolumn_struct_only/mysql/go/models.go create mode 100644 internal/endtoend/testdata/copyfrom_singlecolumn_struct_only/mysql/go/query.sql.go create mode 100644 internal/endtoend/testdata/copyfrom_singlecolumn_struct_only/mysql/query.sql create mode 100644 internal/endtoend/testdata/copyfrom_singlecolumn_struct_only/mysql/schema.sql create mode 100644 internal/endtoend/testdata/copyfrom_singlecolumn_struct_only/mysql/sqlc.json diff --git a/internal/endtoend/testdata/copyfrom_singlecolumn_struct_only/issue.md b/internal/endtoend/testdata/copyfrom_singlecolumn_struct_only/issue.md new file mode 100644 index 0000000000..4838ed77c2 --- /dev/null +++ b/internal/endtoend/testdata/copyfrom_singlecolumn_struct_only/issue.md @@ -0,0 +1 @@ +https://github.com/sqlc-dev/sqlc/issues/3443 \ No newline at end of file diff --git a/internal/endtoend/testdata/copyfrom_singlecolumn_struct_only/mysql/go/copyfrom.go b/internal/endtoend/testdata/copyfrom_singlecolumn_struct_only/mysql/go/copyfrom.go new file mode 100644 index 0000000000..424468be6e --- /dev/null +++ b/internal/endtoend/testdata/copyfrom_singlecolumn_struct_only/mysql/go/copyfrom.go @@ -0,0 +1,50 @@ +// Code generated by sqlc. DO NOT EDIT. +// versions: +// sqlc v1.26.0 +// source: copyfrom.go + +package querytest + +import ( + "context" + "fmt" + "io" + "sync/atomic" + + "github.com/go-sql-driver/mysql" + "github.com/hexon/mysqltsv" +) + +var readerHandlerSequenceForInsertSingleValue uint32 = 1 + +func convertRowsForInsertSingleValue(w *io.PipeWriter, arg []InsertSingleValueParams) { + e := mysqltsv.NewEncoder(w, 1, nil) + for _, row := range arg { + e.AppendValue(row.A) + } + w.CloseWithError(e.Close()) +} + +// InsertSingleValue uses MySQL's LOAD DATA LOCAL INFILE and is not atomic. +// +// Errors and duplicate keys are treated as warnings and insertion will +// continue, even without an error for some cases. Use this in a transaction +// and use SHOW WARNINGS to check for any problems and roll back if you want to. +// +// Check the documentation for more information: +// https://dev.mysql.com/doc/refman/8.0/en/load-data.html#load-data-error-handling +func (q *Queries) InsertSingleValue(ctx context.Context, arg []InsertSingleValueParams) (int64, error) { + pr, pw := io.Pipe() + defer pr.Close() + rh := fmt.Sprintf("InsertSingleValue_%d", atomic.AddUint32(&readerHandlerSequenceForInsertSingleValue, 1)) + mysql.RegisterReaderHandler(rh, func() io.Reader { return pr }) + defer mysql.DeregisterReaderHandler(rh) + go convertRowsForInsertSingleValue(pw, arg) + // The string interpolation is necessary because LOAD DATA INFILE requires + // the file name to be given as a literal string. + result, err := q.db.ExecContext(ctx, fmt.Sprintf("LOAD DATA LOCAL INFILE '%s' INTO TABLE `foo` %s (a)", "Reader::"+rh, mysqltsv.Escaping)) + if err != nil { + return 0, err + } + return result.RowsAffected() +} diff --git a/internal/endtoend/testdata/copyfrom_singlecolumn_struct_only/mysql/go/db.go b/internal/endtoend/testdata/copyfrom_singlecolumn_struct_only/mysql/go/db.go new file mode 100644 index 0000000000..51c2e213bc --- /dev/null +++ b/internal/endtoend/testdata/copyfrom_singlecolumn_struct_only/mysql/go/db.go @@ -0,0 +1,31 @@ +// Code generated by sqlc. DO NOT EDIT. +// versions: +// sqlc v1.26.0 + +package querytest + +import ( + "context" + "database/sql" +) + +type DBTX interface { + ExecContext(context.Context, string, ...interface{}) (sql.Result, error) + PrepareContext(context.Context, string) (*sql.Stmt, error) + QueryContext(context.Context, string, ...interface{}) (*sql.Rows, error) + QueryRowContext(context.Context, string, ...interface{}) *sql.Row +} + +func New(db DBTX) *Queries { + return &Queries{db: db} +} + +type Queries struct { + db DBTX +} + +func (q *Queries) WithTx(tx *sql.Tx) *Queries { + return &Queries{ + db: tx, + } +} diff --git a/internal/endtoend/testdata/copyfrom_singlecolumn_struct_only/mysql/go/models.go b/internal/endtoend/testdata/copyfrom_singlecolumn_struct_only/mysql/go/models.go new file mode 100644 index 0000000000..7e52a8799a --- /dev/null +++ b/internal/endtoend/testdata/copyfrom_singlecolumn_struct_only/mysql/go/models.go @@ -0,0 +1,13 @@ +// Code generated by sqlc. DO NOT EDIT. +// versions: +// sqlc v1.26.0 + +package querytest + +import ( + "database/sql" +) + +type Foo struct { + A sql.NullString +} diff --git a/internal/endtoend/testdata/copyfrom_singlecolumn_struct_only/mysql/go/query.sql.go b/internal/endtoend/testdata/copyfrom_singlecolumn_struct_only/mysql/go/query.sql.go new file mode 100644 index 0000000000..5510b6d39c --- /dev/null +++ b/internal/endtoend/testdata/copyfrom_singlecolumn_struct_only/mysql/go/query.sql.go @@ -0,0 +1,18 @@ +// Code generated by sqlc. DO NOT EDIT. +// versions: +// sqlc v1.26.0 +// source: query.sql + +package querytest + +import ( + "database/sql" +) + +const insertSingleValue = `-- name: InsertSingleValue :copyfrom +INSERT INTO foo (a) VALUES (?) +` + +type InsertSingleValueParams struct { + A sql.NullString +} diff --git a/internal/endtoend/testdata/copyfrom_singlecolumn_struct_only/mysql/query.sql b/internal/endtoend/testdata/copyfrom_singlecolumn_struct_only/mysql/query.sql new file mode 100644 index 0000000000..091c90316b --- /dev/null +++ b/internal/endtoend/testdata/copyfrom_singlecolumn_struct_only/mysql/query.sql @@ -0,0 +1,2 @@ +-- name: InsertSingleValue :copyfrom +INSERT INTO foo (a) VALUES (?); diff --git a/internal/endtoend/testdata/copyfrom_singlecolumn_struct_only/mysql/schema.sql b/internal/endtoend/testdata/copyfrom_singlecolumn_struct_only/mysql/schema.sql new file mode 100644 index 0000000000..022bbd6f91 --- /dev/null +++ b/internal/endtoend/testdata/copyfrom_singlecolumn_struct_only/mysql/schema.sql @@ -0,0 +1 @@ +CREATE TABLE foo (a text); diff --git a/internal/endtoend/testdata/copyfrom_singlecolumn_struct_only/mysql/sqlc.json b/internal/endtoend/testdata/copyfrom_singlecolumn_struct_only/mysql/sqlc.json new file mode 100644 index 0000000000..4a4d43c146 --- /dev/null +++ b/internal/endtoend/testdata/copyfrom_singlecolumn_struct_only/mysql/sqlc.json @@ -0,0 +1,15 @@ +{ + "version": "1", + "packages": [ + { + "path": "go", + "sql_package": "database/sql", + "sql_driver": "github.com/go-sql-driver/mysql", + "engine": "mysql", + "name": "querytest", + "schema": "schema.sql", + "queries": "query.sql", + "query_parameter_limit": 0 + } + ] +} From 462070d9c75a6fc2e71952903b5bd950eb078c10 Mon Sep 17 00:00:00 2001 From: Terri Cain Date: Fri, 14 Jun 2024 22:48:21 +0100 Subject: [PATCH 3/4] Fixed issue where copyfrom structs were not being emitted --- internal/codegen/golang/result.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/internal/codegen/golang/result.go b/internal/codegen/golang/result.go index 560e112af2..515d0a654f 100644 --- a/internal/codegen/golang/result.go +++ b/internal/codegen/golang/result.go @@ -259,7 +259,9 @@ func buildQueries(req *plugin.GenerateRequest, options *opts.Options, structs [] EmitPointer: options.EmitParamsStructPointers, } - if len(query.Params) <= qpl { + // if query params is 2, and query params limit is 4 AND this is a copyfrom, we still want to emit the query's model + // otherwise we end up with a copyfrom using a struct without the struct definition + if len(query.Params) <= qpl && query.Cmd != ":copyfrom" { gq.Arg.Emit = false } } From d6139fd1684feb2d3cc5fd431c60bf50a8d7a8c3 Mon Sep 17 00:00:00 2001 From: Terri Cain Date: Fri, 14 Jun 2024 22:48:57 +0100 Subject: [PATCH 4/4] Added testcase to catch not emitting structs used with copyfrom --- .../mysql/go/copyfrom.go | 51 +++++++++++++++++++ .../mysql/go/db.go | 31 +++++++++++ .../mysql/go/models.go | 14 +++++ .../mysql/go/query.sql.go | 19 +++++++ .../mysql/query.sql | 2 + .../mysql/schema.sql | 1 + .../mysql/sqlc.json | 15 ++++++ 7 files changed, 133 insertions(+) create mode 100644 internal/endtoend/testdata/copyfrom_multicolumn_parameter_limit/mysql/go/copyfrom.go create mode 100644 internal/endtoend/testdata/copyfrom_multicolumn_parameter_limit/mysql/go/db.go create mode 100644 internal/endtoend/testdata/copyfrom_multicolumn_parameter_limit/mysql/go/models.go create mode 100644 internal/endtoend/testdata/copyfrom_multicolumn_parameter_limit/mysql/go/query.sql.go create mode 100644 internal/endtoend/testdata/copyfrom_multicolumn_parameter_limit/mysql/query.sql create mode 100644 internal/endtoend/testdata/copyfrom_multicolumn_parameter_limit/mysql/schema.sql create mode 100644 internal/endtoend/testdata/copyfrom_multicolumn_parameter_limit/mysql/sqlc.json diff --git a/internal/endtoend/testdata/copyfrom_multicolumn_parameter_limit/mysql/go/copyfrom.go b/internal/endtoend/testdata/copyfrom_multicolumn_parameter_limit/mysql/go/copyfrom.go new file mode 100644 index 0000000000..19746198f3 --- /dev/null +++ b/internal/endtoend/testdata/copyfrom_multicolumn_parameter_limit/mysql/go/copyfrom.go @@ -0,0 +1,51 @@ +// Code generated by sqlc. DO NOT EDIT. +// versions: +// sqlc v1.26.0 +// source: copyfrom.go + +package querytest + +import ( + "context" + "fmt" + "io" + "sync/atomic" + + "github.com/go-sql-driver/mysql" + "github.com/hexon/mysqltsv" +) + +var readerHandlerSequenceForInsertMultipleValues uint32 = 1 + +func convertRowsForInsertMultipleValues(w *io.PipeWriter, arg []InsertMultipleValuesParams) { + e := mysqltsv.NewEncoder(w, 2, nil) + for _, row := range arg { + e.AppendValue(row.A) + e.AppendValue(row.B) + } + w.CloseWithError(e.Close()) +} + +// InsertMultipleValues uses MySQL's LOAD DATA LOCAL INFILE and is not atomic. +// +// Errors and duplicate keys are treated as warnings and insertion will +// continue, even without an error for some cases. Use this in a transaction +// and use SHOW WARNINGS to check for any problems and roll back if you want to. +// +// Check the documentation for more information: +// https://dev.mysql.com/doc/refman/8.0/en/load-data.html#load-data-error-handling +func (q *Queries) InsertMultipleValues(ctx context.Context, arg []InsertMultipleValuesParams) (int64, error) { + pr, pw := io.Pipe() + defer pr.Close() + rh := fmt.Sprintf("InsertMultipleValues_%d", atomic.AddUint32(&readerHandlerSequenceForInsertMultipleValues, 1)) + mysql.RegisterReaderHandler(rh, func() io.Reader { return pr }) + defer mysql.DeregisterReaderHandler(rh) + go convertRowsForInsertMultipleValues(pw, arg) + // The string interpolation is necessary because LOAD DATA INFILE requires + // the file name to be given as a literal string. + result, err := q.db.ExecContext(ctx, fmt.Sprintf("LOAD DATA LOCAL INFILE '%s' INTO TABLE `foo` %s (a, b)", "Reader::"+rh, mysqltsv.Escaping)) + if err != nil { + return 0, err + } + return result.RowsAffected() +} diff --git a/internal/endtoend/testdata/copyfrom_multicolumn_parameter_limit/mysql/go/db.go b/internal/endtoend/testdata/copyfrom_multicolumn_parameter_limit/mysql/go/db.go new file mode 100644 index 0000000000..51c2e213bc --- /dev/null +++ b/internal/endtoend/testdata/copyfrom_multicolumn_parameter_limit/mysql/go/db.go @@ -0,0 +1,31 @@ +// Code generated by sqlc. DO NOT EDIT. +// versions: +// sqlc v1.26.0 + +package querytest + +import ( + "context" + "database/sql" +) + +type DBTX interface { + ExecContext(context.Context, string, ...interface{}) (sql.Result, error) + PrepareContext(context.Context, string) (*sql.Stmt, error) + QueryContext(context.Context, string, ...interface{}) (*sql.Rows, error) + QueryRowContext(context.Context, string, ...interface{}) *sql.Row +} + +func New(db DBTX) *Queries { + return &Queries{db: db} +} + +type Queries struct { + db DBTX +} + +func (q *Queries) WithTx(tx *sql.Tx) *Queries { + return &Queries{ + db: tx, + } +} diff --git a/internal/endtoend/testdata/copyfrom_multicolumn_parameter_limit/mysql/go/models.go b/internal/endtoend/testdata/copyfrom_multicolumn_parameter_limit/mysql/go/models.go new file mode 100644 index 0000000000..d29a0fb734 --- /dev/null +++ b/internal/endtoend/testdata/copyfrom_multicolumn_parameter_limit/mysql/go/models.go @@ -0,0 +1,14 @@ +// Code generated by sqlc. DO NOT EDIT. +// versions: +// sqlc v1.26.0 + +package querytest + +import ( + "database/sql" +) + +type Foo struct { + A sql.NullString + B sql.NullString +} diff --git a/internal/endtoend/testdata/copyfrom_multicolumn_parameter_limit/mysql/go/query.sql.go b/internal/endtoend/testdata/copyfrom_multicolumn_parameter_limit/mysql/go/query.sql.go new file mode 100644 index 0000000000..3fb9f22c06 --- /dev/null +++ b/internal/endtoend/testdata/copyfrom_multicolumn_parameter_limit/mysql/go/query.sql.go @@ -0,0 +1,19 @@ +// Code generated by sqlc. DO NOT EDIT. +// versions: +// sqlc v1.26.0 +// source: query.sql + +package querytest + +import ( + "database/sql" +) + +const insertMultipleValues = `-- name: InsertMultipleValues :copyfrom +INSERT INTO foo (a, b) VALUES (?, ?) +` + +type InsertMultipleValuesParams struct { + A sql.NullString + B sql.NullString +} diff --git a/internal/endtoend/testdata/copyfrom_multicolumn_parameter_limit/mysql/query.sql b/internal/endtoend/testdata/copyfrom_multicolumn_parameter_limit/mysql/query.sql new file mode 100644 index 0000000000..18a13f88e6 --- /dev/null +++ b/internal/endtoend/testdata/copyfrom_multicolumn_parameter_limit/mysql/query.sql @@ -0,0 +1,2 @@ +-- name: InsertMultipleValues :copyfrom +INSERT INTO foo (a, b) VALUES (?, ?); diff --git a/internal/endtoend/testdata/copyfrom_multicolumn_parameter_limit/mysql/schema.sql b/internal/endtoend/testdata/copyfrom_multicolumn_parameter_limit/mysql/schema.sql new file mode 100644 index 0000000000..dd2f6b6b50 --- /dev/null +++ b/internal/endtoend/testdata/copyfrom_multicolumn_parameter_limit/mysql/schema.sql @@ -0,0 +1 @@ +CREATE TABLE foo (a text, b text); diff --git a/internal/endtoend/testdata/copyfrom_multicolumn_parameter_limit/mysql/sqlc.json b/internal/endtoend/testdata/copyfrom_multicolumn_parameter_limit/mysql/sqlc.json new file mode 100644 index 0000000000..0990bc2fef --- /dev/null +++ b/internal/endtoend/testdata/copyfrom_multicolumn_parameter_limit/mysql/sqlc.json @@ -0,0 +1,15 @@ +{ + "version": "1", + "packages": [ + { + "path": "go", + "sql_package": "database/sql", + "sql_driver": "github.com/go-sql-driver/mysql", + "engine": "mysql", + "name": "querytest", + "schema": "schema.sql", + "queries": "query.sql", + "query_parameter_limit": 4 + } + ] +}