From 23e86acc2ab20ff92b93c66e2bffc68f9a85a0ff Mon Sep 17 00:00:00 2001 From: Vitaliy Solodilov Date: Wed, 26 Jul 2023 16:50:33 +0300 Subject: [PATCH] feat(sqlite): virtual tables and fts5 supported Fixes: #1797 Out of scope: * table-valued function. https://www.sqlite.org/vtab.html#tabfunc2 * insert commands. https://www.sqlite.org/fts5.html#special_insert_commands * you cannot use table name to the left of MATCH or equality operator --- .../sqlc_slice_prepared/sqlite/go/db.go | 2 +- .../sqlc_slice_prepared/sqlite/go/models.go | 2 +- .../sqlite/go/query.sql.go | 2 +- .../testdata/virtual_table/sqlite/go/db.go | 31 +++ .../virtual_table/sqlite/go/models.go | 26 ++ .../virtual_table/sqlite/go/query.sql.go | 248 ++++++++++++++++++ .../testdata/virtual_table/sqlite/query.sql | 35 +++ .../testdata/virtual_table/sqlite/schema.sql | 15 ++ .../testdata/virtual_table/sqlite/sqlc.yaml | 9 + internal/engine/sqlite/convert.go | 48 ++++ internal/engine/sqlite/stdlib.go | 56 ++++ 11 files changed, 471 insertions(+), 3 deletions(-) create mode 100644 internal/endtoend/testdata/virtual_table/sqlite/go/db.go create mode 100644 internal/endtoend/testdata/virtual_table/sqlite/go/models.go create mode 100644 internal/endtoend/testdata/virtual_table/sqlite/go/query.sql.go create mode 100644 internal/endtoend/testdata/virtual_table/sqlite/query.sql create mode 100644 internal/endtoend/testdata/virtual_table/sqlite/schema.sql create mode 100644 internal/endtoend/testdata/virtual_table/sqlite/sqlc.yaml diff --git a/internal/endtoend/testdata/sqlc_slice_prepared/sqlite/go/db.go b/internal/endtoend/testdata/sqlc_slice_prepared/sqlite/go/db.go index abb75859c5..c4c016bd34 100644 --- a/internal/endtoend/testdata/sqlc_slice_prepared/sqlite/go/db.go +++ b/internal/endtoend/testdata/sqlc_slice_prepared/sqlite/go/db.go @@ -1,6 +1,6 @@ // Code generated by sqlc. DO NOT EDIT. // versions: -// sqlc v1.19.0 +// sqlc v1.19.1 package querytest diff --git a/internal/endtoend/testdata/sqlc_slice_prepared/sqlite/go/models.go b/internal/endtoend/testdata/sqlc_slice_prepared/sqlite/go/models.go index 1aa7431b0e..60f9f395ad 100644 --- a/internal/endtoend/testdata/sqlc_slice_prepared/sqlite/go/models.go +++ b/internal/endtoend/testdata/sqlc_slice_prepared/sqlite/go/models.go @@ -1,6 +1,6 @@ // Code generated by sqlc. DO NOT EDIT. // versions: -// sqlc v1.19.0 +// sqlc v1.19.1 package querytest diff --git a/internal/endtoend/testdata/sqlc_slice_prepared/sqlite/go/query.sql.go b/internal/endtoend/testdata/sqlc_slice_prepared/sqlite/go/query.sql.go index 81774de9c0..c2012892ea 100644 --- a/internal/endtoend/testdata/sqlc_slice_prepared/sqlite/go/query.sql.go +++ b/internal/endtoend/testdata/sqlc_slice_prepared/sqlite/go/query.sql.go @@ -1,6 +1,6 @@ // Code generated by sqlc. DO NOT EDIT. // versions: -// sqlc v1.19.0 +// sqlc v1.19.1 // source: query.sql package querytest diff --git a/internal/endtoend/testdata/virtual_table/sqlite/go/db.go b/internal/endtoend/testdata/virtual_table/sqlite/go/db.go new file mode 100644 index 0000000000..fb6ae669f6 --- /dev/null +++ b/internal/endtoend/testdata/virtual_table/sqlite/go/db.go @@ -0,0 +1,31 @@ +// Code generated by sqlc. DO NOT EDIT. +// versions: +// sqlc v1.19.1 + +package querytest + +import ( + "context" + "database/sql" +) + +type DBTX interface { + ExecContext(context.Context, string, ...interface{}) (sql.Result, error) + PrepareContext(context.Context, string) (*sql.Stmt, error) + QueryContext(context.Context, string, ...interface{}) (*sql.Rows, error) + QueryRowContext(context.Context, string, ...interface{}) *sql.Row +} + +func New(db DBTX) *Queries { + return &Queries{db: db} +} + +type Queries struct { + db DBTX +} + +func (q *Queries) WithTx(tx *sql.Tx) *Queries { + return &Queries{ + db: tx, + } +} diff --git a/internal/endtoend/testdata/virtual_table/sqlite/go/models.go b/internal/endtoend/testdata/virtual_table/sqlite/go/models.go new file mode 100644 index 0000000000..2dbe158815 --- /dev/null +++ b/internal/endtoend/testdata/virtual_table/sqlite/go/models.go @@ -0,0 +1,26 @@ +// Code generated by sqlc. DO NOT EDIT. +// versions: +// sqlc v1.19.1 + +package querytest + +import ( + "database/sql" +) + +type Ft struct { + B string +} + +type Tbl struct { + A int64 + B sql.NullString + C sql.NullString + D sql.NullString + E sql.NullInt64 +} + +type TblFt struct { + B string + C string +} diff --git a/internal/endtoend/testdata/virtual_table/sqlite/go/query.sql.go b/internal/endtoend/testdata/virtual_table/sqlite/go/query.sql.go new file mode 100644 index 0000000000..d558e55b44 --- /dev/null +++ b/internal/endtoend/testdata/virtual_table/sqlite/go/query.sql.go @@ -0,0 +1,248 @@ +// Code generated by sqlc. DO NOT EDIT. +// versions: +// sqlc v1.19.1 +// source: query.sql + +package querytest + +import ( + "context" +) + +const deleteTblFt = `-- name: DeleteTblFt :exec +DELETE FROM tbl_ft WHERE b = ? +` + +func (q *Queries) DeleteTblFt(ctx context.Context, b string) error { + _, err := q.db.ExecContext(ctx, deleteTblFt, b) + return err +} + +const insertTblFt = `-- name: InsertTblFt :exec +INSERT INTO tbl_ft(b, c) VALUES(?, ?) +` + +type InsertTblFtParams struct { + B string + C string +} + +func (q *Queries) InsertTblFt(ctx context.Context, arg InsertTblFtParams) error { + _, err := q.db.ExecContext(ctx, insertTblFt, arg.B, arg.C) + return err +} + +const selectAllColsFt = `-- name: SelectAllColsFt :many +SELECT b FROM ft +WHERE b MATCH ? +` + +func (q *Queries) SelectAllColsFt(ctx context.Context, b string) ([]string, error) { + rows, err := q.db.QueryContext(ctx, selectAllColsFt, b) + if err != nil { + return nil, err + } + defer rows.Close() + var items []string + for rows.Next() { + var b string + if err := rows.Scan(&b); err != nil { + return nil, err + } + items = append(items, b) + } + if err := rows.Close(); err != nil { + return nil, err + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + +const selectAllColsTblFt = `-- name: SelectAllColsTblFt :many +SELECT b, c FROM tbl_ft +WHERE b MATCH ? +` + +func (q *Queries) SelectAllColsTblFt(ctx context.Context, b string) ([]TblFt, error) { + rows, err := q.db.QueryContext(ctx, selectAllColsTblFt, b) + if err != nil { + return nil, err + } + defer rows.Close() + var items []TblFt + for rows.Next() { + var i TblFt + if err := rows.Scan(&i.B, &i.C); err != nil { + return nil, err + } + items = append(items, i) + } + if err := rows.Close(); err != nil { + return nil, err + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + +const selectBm25Func = `-- name: SelectBm25Func :many +SELECT b, c, bm25(tbl_ft, 2.0) FROM tbl_ft +WHERE b MATCH ? ORDER BY bm25(tbl_ft) +` + +type SelectBm25FuncRow struct { + B string + C string + Bm25 float64 +} + +func (q *Queries) SelectBm25Func(ctx context.Context, b string) ([]SelectBm25FuncRow, error) { + rows, err := q.db.QueryContext(ctx, selectBm25Func, b) + if err != nil { + return nil, err + } + defer rows.Close() + var items []SelectBm25FuncRow + for rows.Next() { + var i SelectBm25FuncRow + if err := rows.Scan(&i.B, &i.C, &i.Bm25); err != nil { + return nil, err + } + items = append(items, i) + } + if err := rows.Close(); err != nil { + return nil, err + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + +const selectHightlighFunc = `-- name: SelectHightlighFunc :many +SELECT highlight(tbl_ft, 0, '', '') FROM tbl_ft +WHERE b MATCH ? +` + +func (q *Queries) SelectHightlighFunc(ctx context.Context, b string) ([]string, error) { + rows, err := q.db.QueryContext(ctx, selectHightlighFunc, b) + if err != nil { + return nil, err + } + defer rows.Close() + var items []string + for rows.Next() { + var highlight string + if err := rows.Scan(&highlight); err != nil { + return nil, err + } + items = append(items, highlight) + } + if err := rows.Close(); err != nil { + return nil, err + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + +const selectOneColFt = `-- name: SelectOneColFt :many +SELECT b FROM ft +WHERE b = ? +` + +func (q *Queries) SelectOneColFt(ctx context.Context, b string) ([]string, error) { + rows, err := q.db.QueryContext(ctx, selectOneColFt, b) + if err != nil { + return nil, err + } + defer rows.Close() + var items []string + for rows.Next() { + var b string + if err := rows.Scan(&b); err != nil { + return nil, err + } + items = append(items, b) + } + if err := rows.Close(); err != nil { + return nil, err + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + +const selectOneColTblFt = `-- name: SelectOneColTblFt :many +SELECT c FROM tbl_ft +WHERE b = ? +` + +func (q *Queries) SelectOneColTblFt(ctx context.Context, b string) ([]string, error) { + rows, err := q.db.QueryContext(ctx, selectOneColTblFt, b) + if err != nil { + return nil, err + } + defer rows.Close() + var items []string + for rows.Next() { + var c string + if err := rows.Scan(&c); err != nil { + return nil, err + } + items = append(items, c) + } + if err := rows.Close(); err != nil { + return nil, err + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + +const selectSnippetFunc = `-- name: SelectSnippetFunc :many +SELECT snippet(tbl_ft, 0, '', '', 'aa', ?) FROM tbl_ft +` + +func (q *Queries) SelectSnippetFunc(ctx context.Context, snippet int64) ([]string, error) { + rows, err := q.db.QueryContext(ctx, selectSnippetFunc, snippet) + if err != nil { + return nil, err + } + defer rows.Close() + var items []string + for rows.Next() { + var snippet string + if err := rows.Scan(&snippet); err != nil { + return nil, err + } + items = append(items, snippet) + } + if err := rows.Close(); err != nil { + return nil, err + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + +const updateTblFt = `-- name: UpdateTblFt :exec +UPDATE tbl_ft SET c = ? WHERE b = ? +` + +type UpdateTblFtParams struct { + C string + B string +} + +func (q *Queries) UpdateTblFt(ctx context.Context, arg UpdateTblFtParams) error { + _, err := q.db.ExecContext(ctx, updateTblFt, arg.C, arg.B) + return err +} diff --git a/internal/endtoend/testdata/virtual_table/sqlite/query.sql b/internal/endtoend/testdata/virtual_table/sqlite/query.sql new file mode 100644 index 0000000000..ad8eeeae40 --- /dev/null +++ b/internal/endtoend/testdata/virtual_table/sqlite/query.sql @@ -0,0 +1,35 @@ +-- name: SelectAllColsFt :many +SELECT b FROM ft +WHERE b MATCH ?; + +-- name: SelectAllColsTblFt :many +SELECT b, c FROM tbl_ft +WHERE b MATCH ?; + +-- name: SelectOneColFt :many +SELECT b FROM ft +WHERE b = ?; + +-- name: SelectOneColTblFt :many +SELECT c FROM tbl_ft +WHERE b = ?; + +-- name: SelectHightlighFunc :many +SELECT highlight(tbl_ft, 0, '', '') FROM tbl_ft +WHERE b MATCH ?; + +-- name: SelectSnippetFunc :many +SELECT snippet(tbl_ft, 0, '', '', 'aa', ?) FROM tbl_ft; + +-- name: SelectBm25Func :many +SELECT *, bm25(tbl_ft, 2.0) FROM tbl_ft +WHERE b MATCH ? ORDER BY bm25(tbl_ft); + +-- name: UpdateTblFt :exec +UPDATE tbl_ft SET c = ? WHERE b = ?; + +-- name: DeleteTblFt :exec +DELETE FROM tbl_ft WHERE b = ?; + +-- name: InsertTblFt :exec +INSERT INTO tbl_ft(b, c) VALUES(?, ?); diff --git a/internal/endtoend/testdata/virtual_table/sqlite/schema.sql b/internal/endtoend/testdata/virtual_table/sqlite/schema.sql new file mode 100644 index 0000000000..5d3e0a3f8b --- /dev/null +++ b/internal/endtoend/testdata/virtual_table/sqlite/schema.sql @@ -0,0 +1,15 @@ +CREATE TABLE tbl(a INTEGER PRIMARY KEY, b TEXT, c TEXT, d TEXT, e INTEGER); + +CREATE VIRTUAL TABLE tbl_ft USING fts5(b, c UNINDEXED, content='tbl', content_rowid='a'); + +CREATE VIRTUAL TABLE ft USING fts5(b); + +CREATE TRIGGER tbl_ai AFTER INSERT ON tbl BEGIN + INSERT INTO tbl_ft(rowid, b, c) VALUES (new.a, new.b, new.c); +END; + +INSERT INTO tbl VALUES(1, 'xx yy cc', 't', 'a', 11); +INSERT INTO tbl VALUES(2, 'aa bb', 't', 'a', 22); + +INSERT INTO ft VALUES('xx cc'); +INSERT INTO ft VALUES('cc bb'); diff --git a/internal/endtoend/testdata/virtual_table/sqlite/sqlc.yaml b/internal/endtoend/testdata/virtual_table/sqlite/sqlc.yaml new file mode 100644 index 0000000000..50f9bbaab3 --- /dev/null +++ b/internal/endtoend/testdata/virtual_table/sqlite/sqlc.yaml @@ -0,0 +1,9 @@ +version: '2' +sql: +- schema: schema.sql + queries: query.sql + engine: sqlite + gen: + go: + package: querytest + out: go \ No newline at end of file diff --git a/internal/engine/sqlite/convert.go b/internal/engine/sqlite/convert.go index de2437e487..64ff672d6c 100644 --- a/internal/engine/sqlite/convert.go +++ b/internal/engine/sqlite/convert.go @@ -1,6 +1,7 @@ package sqlite import ( + "fmt" "log" "strconv" "strings" @@ -123,6 +124,50 @@ func (c *cc) convertCreate_table_stmtContext(n *parser.Create_table_stmtContext) return stmt } +func (c *cc) convertCreate_virtual_table_stmtContext(n *parser.Create_virtual_table_stmtContext) ast.Node { + switch moduleName := n.Module_name().GetText(); moduleName { + case "fts5": + // https://www.sqlite.org/fts5.html + return c.convertCreate_virtual_table_fts5(n) + default: + return todo( + fmt.Sprintf("create_virtual_table. unsupported module name: %q", moduleName), + n, + ) + } +} + +func (c *cc) convertCreate_virtual_table_fts5(n *parser.Create_virtual_table_stmtContext) ast.Node { + stmt := &ast.CreateTableStmt{ + Name: parseTableName(n), + IfNotExists: n.EXISTS_() != nil, + } + + for _, arg := range n.AllModule_argument() { + var columnName string + + // For example: CREATE VIRTUAL TABLE tbl_ft USING fts5(b, c UNINDEXED) + // * the 'b' column is parsed like Expr_qualified_column_nameContext + // * the 'c' column is parsed like Column_defContext + if columnExpr, ok := arg.Expr().(*parser.Expr_qualified_column_nameContext); ok { + columnName = columnExpr.Column_name().GetText() + } else if columnDef, ok := arg.Column_def().(*parser.Column_defContext); ok { + columnName = columnDef.Column_name().GetText() + } + + if columnName != "" { + stmt.Cols = append(stmt.Cols, &ast.ColumnDef{ + Colname: identifier(columnName), + // you can not specify any column constraints in fts5, so we pass them manually + IsNotNull: true, + TypeName: &ast.TypeName{Name: "text"}, + }) + } + } + + return stmt +} + func (c *cc) convertCreate_view_stmtContext(n *parser.Create_view_stmtContext) ast.Node { viewName := n.View_name().GetText() relation := &ast.RangeVar{ @@ -940,6 +985,9 @@ func (c *cc) convert(node node) ast.Node { case *parser.Create_table_stmtContext: return c.convertCreate_table_stmtContext(n) + case *parser.Create_virtual_table_stmtContext: + return c.convertCreate_virtual_table_stmtContext(n) + case *parser.Create_view_stmtContext: return c.convertCreate_view_stmtContext(n) diff --git a/internal/engine/sqlite/stdlib.go b/internal/engine/sqlite/stdlib.go index 82dadf06c5..9c30eb9b14 100644 --- a/internal/engine/sqlite/stdlib.go +++ b/internal/engine/sqlite/stdlib.go @@ -924,6 +924,62 @@ func defaultSchema(name string) *catalog.Schema { }, ReturnType: &ast.TypeName{Name: "blob"}, }, + // fts5 funcs https://www.sqlite.org/fts5.html#_auxiliary_functions_ + { + Name: "HIGHLIGHT", + Args: []*catalog.Argument{ + { + Type: &ast.TypeName{Name: "text"}, + }, + { + Type: &ast.TypeName{Name: "integer"}, + }, + { + Type: &ast.TypeName{Name: "text"}, + }, + { + Type: &ast.TypeName{Name: "text"}, + }, + }, + ReturnType: &ast.TypeName{Name: "text"}, + }, + { + Name: "SNIPPET", + Args: []*catalog.Argument{ + { + Type: &ast.TypeName{Name: "text"}, + }, + { + Type: &ast.TypeName{Name: "integer"}, + }, + { + Type: &ast.TypeName{Name: "text"}, + }, + { + Type: &ast.TypeName{Name: "text"}, + }, + { + Type: &ast.TypeName{Name: "text"}, + }, + { + Type: &ast.TypeName{Name: "integer"}, + }, + }, + ReturnType: &ast.TypeName{Name: "text"}, + }, + { + Name: "bm25", + Args: []*catalog.Argument{ + { + Type: &ast.TypeName{Name: "text"}, + }, + { + Type: &ast.TypeName{Name: "real"}, + Mode: ast.FuncParamVariadic, + }, + }, + ReturnType: &ast.TypeName{Name: "real"}, + }, } return s }