From 3695ee3a8c2d9340dd71940152b65fed02b2a564 Mon Sep 17 00:00:00 2001 From: Brandur Date: Thu, 15 May 2025 23:03:34 -0700 Subject: [PATCH] SQLite: Coerce jsonb columns to json before returning to Go code This one follows up the discussion in #3953 to try and make the `jsonb` data type in SQLite usable (see discussion there, but I believe that it's currently not). According the SQLite docs on JSONB [1], it's considered a format that's internal to the database itself, and no attempt should be made to parse it elsewhere: > JSONB is not intended as an external format to be used by > applications. JSONB is designed for internal use by SQLite only. > Programmers do not need to understand the JSONB format in order to use > it effectively. Applications should access JSONB only through the JSON > SQL functions, not by looking at individual bytes of the BLOB. Currently, when trying to use a `jsonb` column in SQLite, sqlc ends up returning the internal binary data, which ends up being unparsable in Go: riverdrivertest.go:3030: Error Trace: /Users/brandur/Documents/projects/river/internal/riverinternaltest/riverdrivertest/riverdrivertest.go:3030 Error: Not equal: expected: []byte{0x7b, 0x22, 0x66, 0x6f, 0x6f, 0x22, 0x3a, 0x20, 0x22, 0x62, 0x61, 0x72, 0x22, 0x7d} actual : []byte{0x8c, 0x37, 0x66, 0x6f, 0x6f, 0x37, 0x62, 0x61, 0x72} Diff: --- Expected +++ Actual @@ -1,3 +1,3 @@ -([]uint8) (len=14) { - 00000000 7b 22 66 6f 6f 22 3a 20 22 62 61 72 22 7d |{"foo": "bar"}| +([]uint8) (len=9) { + 00000000 8c 37 66 6f 6f 37 62 61 72 |.7foo7bar| } Test: TestDriverRiverSQLite/QueueCreateOrSetUpdatedAt/InsertsANewQueueWithDefaultUpdatedAt The fix is that we should make sure to coerce `jsonb` columns back to `json` before returning. That's what this pull request does, intercepting `SELECT *` and wrapping `jsonb` columns with a `json(...)` invocation. I also assign `json` and `jsonb` a `[]byte` data type by default so they don't end up as `any`, which isn't very useful. `[]byte` is consistent with the default for `pgx/v5`. [1] https://sqlite.org/jsonb.html --- internal/codegen/golang/sqlite_type.go | 3 ++ internal/compiler/expand.go | 12 +++++ internal/endtoend/testdata/jsonb/pgx/go/db.go | 32 ++++++++++++ .../endtoend/testdata/jsonb/pgx/go/models.go | 12 +++++ .../testdata/jsonb/pgx/go/query.sql.go | 50 +++++++++++++++++++ .../endtoend/testdata/jsonb/pgx/query.sql | 15 ++++++ .../endtoend/testdata/jsonb/pgx/schema.sql | 7 +++ .../endtoend/testdata/jsonb/pgx/sqlc.json | 13 +++++ .../endtoend/testdata/jsonb/sqlite/go/db.go | 31 ++++++++++++ .../testdata/jsonb/sqlite/go/models.go | 12 +++++ .../testdata/jsonb/sqlite/go/query.sql.go | 50 +++++++++++++++++++ .../endtoend/testdata/jsonb/sqlite/query.sql | 15 ++++++ .../endtoend/testdata/jsonb/sqlite/schema.sql | 7 +++ .../endtoend/testdata/jsonb/sqlite/sqlc.json | 12 +++++ 14 files changed, 271 insertions(+) create mode 100644 internal/endtoend/testdata/jsonb/pgx/go/db.go create mode 100644 internal/endtoend/testdata/jsonb/pgx/go/models.go create mode 100644 internal/endtoend/testdata/jsonb/pgx/go/query.sql.go create mode 100644 internal/endtoend/testdata/jsonb/pgx/query.sql create mode 100644 internal/endtoend/testdata/jsonb/pgx/schema.sql create mode 100644 internal/endtoend/testdata/jsonb/pgx/sqlc.json create mode 100644 internal/endtoend/testdata/jsonb/sqlite/go/db.go create mode 100644 internal/endtoend/testdata/jsonb/sqlite/go/models.go create mode 100644 internal/endtoend/testdata/jsonb/sqlite/go/query.sql.go create mode 100644 internal/endtoend/testdata/jsonb/sqlite/query.sql create mode 100644 internal/endtoend/testdata/jsonb/sqlite/schema.sql create mode 100644 internal/endtoend/testdata/jsonb/sqlite/sqlc.json diff --git a/internal/codegen/golang/sqlite_type.go b/internal/codegen/golang/sqlite_type.go index 92c13557f6..cb7348385f 100644 --- a/internal/codegen/golang/sqlite_type.go +++ b/internal/codegen/golang/sqlite_type.go @@ -56,6 +56,9 @@ func sqliteType(req *plugin.GenerateRequest, options *opts.Options, col *plugin. } return "sql.NullTime" + case "json", "jsonb": + return "[]byte" + case "any": return "interface{}" diff --git a/internal/compiler/expand.go b/internal/compiler/expand.go index 60e654b696..2f04e9a6f1 100644 --- a/internal/compiler/expand.go +++ b/internal/compiler/expand.go @@ -6,6 +6,7 @@ import ( "strings" "github.com/sqlc-dev/sqlc/internal/config" + "github.com/sqlc-dev/sqlc/internal/engine/sqlite" "github.com/sqlc-dev/sqlc/internal/source" "github.com/sqlc-dev/sqlc/internal/sql/ast" "github.com/sqlc-dev/sqlc/internal/sql/astutils" @@ -149,6 +150,17 @@ func (c *Compiler) expandStmt(qc *QueryCatalog, raw *ast.RawStmt, node ast.Node) if counts[cname] > 1 { cname = tableName + "." + cname } + // Under SQLite, neither json nor jsonb are real data types, and + // rather just of type blob, so database drivers just return + // whatever raw binary is stored as values. This is a problem + // for jsonb, which is considered an internal format to SQLite + // and no attempt should be made to parse it outside of the + // database itself. For jsonb columns in SQLite, wrap returned + // columns in `json(col)` to coerce the internal binary format + // to JSON parsable by the user-space application. + if _, ok := c.parser.(*sqlite.Parser); ok && column.DataType == "jsonb" { + cname = "json(" + cname + ")" + } cols = append(cols, cname) } } diff --git a/internal/endtoend/testdata/jsonb/pgx/go/db.go b/internal/endtoend/testdata/jsonb/pgx/go/db.go new file mode 100644 index 0000000000..e83d6a948c --- /dev/null +++ b/internal/endtoend/testdata/jsonb/pgx/go/db.go @@ -0,0 +1,32 @@ +// Code generated by sqlc. DO NOT EDIT. +// versions: +// sqlc v1.29.0 + +package querytest + +import ( + "context" + + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgconn" +) + +type DBTX interface { + Exec(context.Context, string, ...interface{}) (pgconn.CommandTag, error) + Query(context.Context, string, ...interface{}) (pgx.Rows, error) + QueryRow(context.Context, string, ...interface{}) pgx.Row +} + +func New(db DBTX) *Queries { + return &Queries{db: db} +} + +type Queries struct { + db DBTX +} + +func (q *Queries) WithTx(tx pgx.Tx) *Queries { + return &Queries{ + db: tx, + } +} diff --git a/internal/endtoend/testdata/jsonb/pgx/go/models.go b/internal/endtoend/testdata/jsonb/pgx/go/models.go new file mode 100644 index 0000000000..1932ce7f53 --- /dev/null +++ b/internal/endtoend/testdata/jsonb/pgx/go/models.go @@ -0,0 +1,12 @@ +// Code generated by sqlc. DO NOT EDIT. +// versions: +// sqlc v1.29.0 + +package querytest + +type Foo struct { + A []byte + B []byte + C []byte + D []byte +} diff --git a/internal/endtoend/testdata/jsonb/pgx/go/query.sql.go b/internal/endtoend/testdata/jsonb/pgx/go/query.sql.go new file mode 100644 index 0000000000..1d7532f6ec --- /dev/null +++ b/internal/endtoend/testdata/jsonb/pgx/go/query.sql.go @@ -0,0 +1,50 @@ +// Code generated by sqlc. DO NOT EDIT. +// versions: +// sqlc v1.29.0 +// source: query.sql + +package querytest + +import ( + "context" +) + +const insertFoo = `-- name: InsertFoo :exec +INSERT INTO foo ( + a, + b, + c, + d +) VALUES ( + $1, + $2, + $3, + $4 +) RETURNING a, b, c, d +` + +type InsertFooParams struct { + A []byte + B []byte + C []byte + D []byte +} + +func (q *Queries) InsertFoo(ctx context.Context, arg InsertFooParams) error { + _, err := q.db.Exec(ctx, insertFoo, + arg.A, + arg.B, + arg.C, + arg.D, + ) + return err +} + +const selectFoo = `-- name: SelectFoo :exec +SELECT a, b, c, d FROM foo +` + +func (q *Queries) SelectFoo(ctx context.Context) error { + _, err := q.db.Exec(ctx, selectFoo) + return err +} diff --git a/internal/endtoend/testdata/jsonb/pgx/query.sql b/internal/endtoend/testdata/jsonb/pgx/query.sql new file mode 100644 index 0000000000..6959bd1a70 --- /dev/null +++ b/internal/endtoend/testdata/jsonb/pgx/query.sql @@ -0,0 +1,15 @@ +-- name: InsertFoo :exec +INSERT INTO foo ( + a, + b, + c, + d +) VALUES ( + @a, + @b, + @c, + @d +) RETURNING *; + +-- name: SelectFoo :exec +SELECT * FROM foo; diff --git a/internal/endtoend/testdata/jsonb/pgx/schema.sql b/internal/endtoend/testdata/jsonb/pgx/schema.sql new file mode 100644 index 0000000000..6b4a1bb0fd --- /dev/null +++ b/internal/endtoend/testdata/jsonb/pgx/schema.sql @@ -0,0 +1,7 @@ +CREATE TABLE foo ( + a json not null, + b jsonb not null, + c json, + d jsonb +); + diff --git a/internal/endtoend/testdata/jsonb/pgx/sqlc.json b/internal/endtoend/testdata/jsonb/pgx/sqlc.json new file mode 100644 index 0000000000..32ede07158 --- /dev/null +++ b/internal/endtoend/testdata/jsonb/pgx/sqlc.json @@ -0,0 +1,13 @@ +{ + "version": "1", + "packages": [ + { + "path": "go", + "engine": "postgresql", + "sql_package": "pgx/v5", + "name": "querytest", + "schema": "schema.sql", + "queries": "query.sql" + } + ] +} diff --git a/internal/endtoend/testdata/jsonb/sqlite/go/db.go b/internal/endtoend/testdata/jsonb/sqlite/go/db.go new file mode 100644 index 0000000000..a92cd6e8eb --- /dev/null +++ b/internal/endtoend/testdata/jsonb/sqlite/go/db.go @@ -0,0 +1,31 @@ +// Code generated by sqlc. DO NOT EDIT. +// versions: +// sqlc v1.29.0 + +package querytest + +import ( + "context" + "database/sql" +) + +type DBTX interface { + ExecContext(context.Context, string, ...interface{}) (sql.Result, error) + PrepareContext(context.Context, string) (*sql.Stmt, error) + QueryContext(context.Context, string, ...interface{}) (*sql.Rows, error) + QueryRowContext(context.Context, string, ...interface{}) *sql.Row +} + +func New(db DBTX) *Queries { + return &Queries{db: db} +} + +type Queries struct { + db DBTX +} + +func (q *Queries) WithTx(tx *sql.Tx) *Queries { + return &Queries{ + db: tx, + } +} diff --git a/internal/endtoend/testdata/jsonb/sqlite/go/models.go b/internal/endtoend/testdata/jsonb/sqlite/go/models.go new file mode 100644 index 0000000000..1932ce7f53 --- /dev/null +++ b/internal/endtoend/testdata/jsonb/sqlite/go/models.go @@ -0,0 +1,12 @@ +// Code generated by sqlc. DO NOT EDIT. +// versions: +// sqlc v1.29.0 + +package querytest + +type Foo struct { + A []byte + B []byte + C []byte + D []byte +} diff --git a/internal/endtoend/testdata/jsonb/sqlite/go/query.sql.go b/internal/endtoend/testdata/jsonb/sqlite/go/query.sql.go new file mode 100644 index 0000000000..fdde010458 --- /dev/null +++ b/internal/endtoend/testdata/jsonb/sqlite/go/query.sql.go @@ -0,0 +1,50 @@ +// Code generated by sqlc. DO NOT EDIT. +// versions: +// sqlc v1.29.0 +// source: query.sql + +package querytest + +import ( + "context" +) + +const insertFoo = `-- name: InsertFoo :exec +INSERT INTO foo ( + a, + b, + c, + d +) VALUES ( + ?1, + ?2, + ?3, + ?4 +) RETURNING a, json(b), c, json(d) +` + +type InsertFooParams struct { + A []byte + B []byte + C []byte + D []byte +} + +func (q *Queries) InsertFoo(ctx context.Context, arg InsertFooParams) error { + _, err := q.db.ExecContext(ctx, insertFoo, + arg.A, + arg.B, + arg.C, + arg.D, + ) + return err +} + +const selectFoo = `-- name: SelectFoo :exec +SELECT a, json(b), c, json(d) FROM foo +` + +func (q *Queries) SelectFoo(ctx context.Context) error { + _, err := q.db.ExecContext(ctx, selectFoo) + return err +} diff --git a/internal/endtoend/testdata/jsonb/sqlite/query.sql b/internal/endtoend/testdata/jsonb/sqlite/query.sql new file mode 100644 index 0000000000..6959bd1a70 --- /dev/null +++ b/internal/endtoend/testdata/jsonb/sqlite/query.sql @@ -0,0 +1,15 @@ +-- name: InsertFoo :exec +INSERT INTO foo ( + a, + b, + c, + d +) VALUES ( + @a, + @b, + @c, + @d +) RETURNING *; + +-- name: SelectFoo :exec +SELECT * FROM foo; diff --git a/internal/endtoend/testdata/jsonb/sqlite/schema.sql b/internal/endtoend/testdata/jsonb/sqlite/schema.sql new file mode 100644 index 0000000000..6b4a1bb0fd --- /dev/null +++ b/internal/endtoend/testdata/jsonb/sqlite/schema.sql @@ -0,0 +1,7 @@ +CREATE TABLE foo ( + a json not null, + b jsonb not null, + c json, + d jsonb +); + diff --git a/internal/endtoend/testdata/jsonb/sqlite/sqlc.json b/internal/endtoend/testdata/jsonb/sqlite/sqlc.json new file mode 100644 index 0000000000..cd66df063b --- /dev/null +++ b/internal/endtoend/testdata/jsonb/sqlite/sqlc.json @@ -0,0 +1,12 @@ +{ + "version": "1", + "packages": [ + { + "path": "go", + "engine": "sqlite", + "name": "querytest", + "schema": "schema.sql", + "queries": "query.sql" + } + ] +}