From d6ff737c755f3bba5a7cb65b6d1c077c3b32786b Mon Sep 17 00:00:00 2001 From: "Zephaniah E. Loss-Cutler-Hull" Date: Mon, 25 Jan 2021 21:41:29 -0800 Subject: [PATCH 1/8] Add support for globs in overrides. This lets us say things like '*.*_ksuid' should be of type 'github.com/segmentio/ksuid.KSUID', instead of having to list every single column in every single table. We can even say '*.*.*_ksuid' for every column, in every table, in every schema. Now, ideally we would do a type replace on 'binary(20)', however that gets turned into a char for at least the mysql code, which means that we never get to see 'binary(20)', and so can't do a replacement on it. This makes use of the github.com/gobwas/glob package for the globbing, and it pretty much Just Works the way you expect it to. --- go.mod | 1 + go.sum | 2 ++ internal/codegen/golang/compat.go | 2 +- internal/codegen/golang/go_type.go | 2 +- internal/codegen/golang/result.go | 4 +++- internal/codegen/kotlin/gen.go | 6 ++++-- internal/config/config.go | 18 ++++++++++-------- internal/core/fqn.go | 6 ++++-- 8 files changed, 26 insertions(+), 15 deletions(-) diff --git a/go.mod b/go.mod index 733b5e51fe..226b6e20d1 100644 --- a/go.mod +++ b/go.mod @@ -6,6 +6,7 @@ require ( github.com/antlr/antlr4 v0.0.0-20200209180723-1177c0b58d07 github.com/davecgh/go-spew v1.1.1 github.com/go-sql-driver/mysql v1.6.0 + github.com/gobwas/glob v0.2.3 github.com/google/go-cmp v0.5.5 github.com/jackc/pgx/v4 v4.11.0 github.com/jinzhu/inflection v1.0.0 diff --git a/go.sum b/go.sum index 6f03a7da11..ade8f04753 100644 --- a/go.sum +++ b/go.sum @@ -101,6 +101,8 @@ github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LB github.com/go-sql-driver/mysql v1.6.0 h1:BCTh4TKNUYmOmMUcQ3IipzF5prigylS7XXjEkfCHuOE= github.com/go-sql-driver/mysql v1.6.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= +github.com/gobwas/glob v0.2.3 h1:A4xDbljILXROh+kObIiy5kIaPYD8e96x1tgBhUI5J+Y= +github.com/gobwas/glob v0.2.3/go.mod h1:d3Ez4x06l9bZtSvzIay5+Yzi0fmZzPgnTbPcKjJAkT8= github.com/gofrs/uuid v3.2.0+incompatible h1:y12jRkkFxsd7GpqdSZ+/KCs/fJbqpEXSGd4+jfEaewE= github.com/gofrs/uuid v3.2.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= github.com/gogo/googleapis v1.1.0/go.mod h1:gf4bu3Q80BeJ6H1S1vYPm8/ELATdvryBaNFGgqEef3s= diff --git a/internal/codegen/golang/compat.go b/internal/codegen/golang/compat.go index cdbb5aac2a..aa38ea3245 100644 --- a/internal/codegen/golang/compat.go +++ b/internal/codegen/golang/compat.go @@ -13,5 +13,5 @@ func sameTableName(n *ast.TableName, f core.FQN, defaultSchema string) bool { if n.Schema == "" { schema = defaultSchema } - return n.Catalog == f.Catalog && schema == f.Schema && n.Name == f.Rel + return n.Catalog == f.Catalog && ((f.Schema != nil && f.Schema.Match(schema)) || schema == "") && ((f.Rel != nil && f.Rel.Match(n.Name)) || n.Name == "") } diff --git a/internal/codegen/golang/go_type.go b/internal/codegen/golang/go_type.go index 11c4f96dbd..16dbde23df 100644 --- a/internal/codegen/golang/go_type.go +++ b/internal/codegen/golang/go_type.go @@ -12,7 +12,7 @@ func goType(r *compiler.Result, col *compiler.Column, settings config.CombinedSe continue } sameTable := sameTableName(col.Table, oride.Table, r.Catalog.DefaultSchema) - if oride.Column != "" && oride.ColumnName == col.Name && sameTable { + if oride.Column != "" && oride.ColumnName.Match(col.Name) && sameTable { return oride.GoTypeName } } diff --git a/internal/codegen/golang/result.go b/internal/codegen/golang/result.go index cbce7d4a2e..640c0480f8 100644 --- a/internal/codegen/golang/result.go +++ b/internal/codegen/golang/result.go @@ -5,6 +5,8 @@ import ( "sort" "strings" + "github.com/gobwas/glob" + "github.com/kyleconroy/sqlc/internal/codegen" "github.com/kyleconroy/sqlc/internal/compiler" "github.com/kyleconroy/sqlc/internal/config" @@ -74,7 +76,7 @@ func buildStructs(r *compiler.Result, settings config.CombinedSettings) []Struct structName = inflection.Singular(structName) } s := Struct{ - Table: core.FQN{Schema: schema.Name, Rel: table.Rel.Name}, + Table: core.FQN{Schema: glob.MustCompile(schema.Name), Rel: glob.MustCompile(table.Rel.Name)}, Name: StructName(structName, settings), Comment: table.Comment, } diff --git a/internal/codegen/kotlin/gen.go b/internal/codegen/kotlin/gen.go index 54cd2a5413..33b7531a77 100644 --- a/internal/codegen/kotlin/gen.go +++ b/internal/codegen/kotlin/gen.go @@ -9,6 +9,8 @@ import ( "strings" "text/template" + "github.com/gobwas/glob" + "github.com/kyleconroy/sqlc/internal/codegen" "github.com/kyleconroy/sqlc/internal/compiler" "github.com/kyleconroy/sqlc/internal/config" @@ -26,7 +28,7 @@ func sameTableName(n *ast.TableName, f core.FQN) bool { if n.Schema == "" { schema = "public" } - return n.Catalog == n.Catalog && schema == f.Schema && n.Name == f.Rel + return n.Catalog == f.Catalog && ((f.Schema != nil && f.Schema.Match(schema)) || schema == "") && ((f.Rel != nil && f.Rel.Match(n.Name)) || n.Name == "") } var ktIdentPattern = regexp.MustCompile("[^a-zA-Z0-9_]+") @@ -286,7 +288,7 @@ func buildDataClasses(r *compiler.Result, settings config.CombinedSettings) []St structName = inflection.Singular(structName) } s := Struct{ - Table: core.FQN{Schema: schema.Name, Rel: table.Rel.Name}, + Table: core.FQN{Schema: glob.MustCompile(schema.Name), Rel: glob.MustCompile(table.Rel.Name)}, Name: structName, Comment: table.Comment, } diff --git a/internal/config/config.go b/internal/config/config.go index bdfba2d28d..775a8d9764 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -11,6 +11,8 @@ import ( yaml "gopkg.in/yaml.v3" + "github.com/gobwas/glob" + "github.com/kyleconroy/sqlc/internal/core" ) @@ -160,7 +162,7 @@ type Override struct { // fully qualified name of the column, e.g. `accounts.id` Column string `json:"column" yaml:"column"` - ColumnName string + ColumnName glob.Glob Table core.FQN GoImportPath string GoPackage string @@ -198,16 +200,16 @@ func (o *Override) Parse() error { colParts := strings.Split(o.Column, ".") switch len(colParts) { case 2: - o.ColumnName = colParts[1] - o.Table = core.FQN{Schema: "public", Rel: colParts[0]} + o.ColumnName = glob.MustCompile(colParts[1]) + o.Table = core.FQN{Schema: glob.MustCompile("public"), Rel: glob.MustCompile(colParts[0])} case 3: - o.ColumnName = colParts[2] - o.Table = core.FQN{Schema: colParts[0], Rel: colParts[1]} + o.ColumnName = glob.MustCompile(colParts[2]) + o.Table = core.FQN{Schema: glob.MustCompile(colParts[0]), Rel: glob.MustCompile(colParts[1])} case 4: - o.ColumnName = colParts[3] - o.Table = core.FQN{Catalog: colParts[0], Schema: colParts[1], Rel: colParts[2]} + o.ColumnName = glob.MustCompile(colParts[3]) + o.Table = core.FQN{Catalog: colParts[0], Schema: glob.MustCompile(colParts[1]), Rel: glob.MustCompile(colParts[2])} default: - return fmt.Errorf("Override `column` specifier %q is not the proper format, expected '[catalog.][schema.]colname.tablename'", o.Column) + return fmt.Errorf("Override `column` specifier %q is not the proper format, expected '[catalog.][schema.]tablename.colname'", o.Column) } } diff --git a/internal/core/fqn.go b/internal/core/fqn.go index 5d76342787..93c7d87246 100644 --- a/internal/core/fqn.go +++ b/internal/core/fqn.go @@ -1,11 +1,13 @@ package core +import "github.com/gobwas/glob" + // TODO: This is the last struct left over from the old architecture. Figure // out how to remove it at some point type FQN struct { Catalog string - Schema string - Rel string + Schema glob.Glob + Rel glob.Glob } func (f FQN) String() string { From 4085b4555b0349e3ea9d476e1d3a3de4ffb1ba72 Mon Sep 17 00:00:00 2001 From: "Zephaniah E. Loss-Cutler-Hull" Date: Mon, 25 Jan 2021 21:44:35 -0800 Subject: [PATCH 2/8] Remove the unused String method on FQN. This doesn't get used, and so we can get rid of it instead of adapting it for globs. --- internal/core/fqn.go | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/internal/core/fqn.go b/internal/core/fqn.go index 93c7d87246..a23402fffb 100644 --- a/internal/core/fqn.go +++ b/internal/core/fqn.go @@ -9,14 +9,3 @@ type FQN struct { Schema glob.Glob Rel glob.Glob } - -func (f FQN) String() string { - s := f.Rel - if f.Schema != "" { - s = f.Schema + "." + s - } - if f.Catalog != "" { - s = f.Catalog + "." + s - } - return s -} From df81aa1296ed99d78c259d0fb32fbd95606ac71a Mon Sep 17 00:00:00 2001 From: "Zephaniah E. Loss-Cutler-Hull" Date: Wed, 7 Apr 2021 16:30:22 -0700 Subject: [PATCH 3/8] Glob support for internal/codegen/python/gen. Fairly straight forward, same changes as for the other code generators. --- internal/codegen/python/gen.go | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/internal/codegen/python/gen.go b/internal/codegen/python/gen.go index a30dd3ef50..7a145d5718 100644 --- a/internal/codegen/python/gen.go +++ b/internal/codegen/python/gen.go @@ -4,6 +4,7 @@ import ( "bufio" "bytes" "fmt" + "github.com/gobwas/glob" "github.com/kyleconroy/sqlc/internal/codegen" "github.com/kyleconroy/sqlc/internal/compiler" "github.com/kyleconroy/sqlc/internal/config" @@ -182,7 +183,7 @@ func pyInnerType(r *compiler.Result, col *compiler.Column, settings config.Combi continue } sameTable := sameTableName(col.Table, oride.Table, r.Catalog.DefaultSchema) - if oride.Column != "" && oride.ColumnName == col.Name && sameTable { + if oride.Column != "" && oride.ColumnName.Match(col.Name) && sameTable { return oride.PythonType.TypeString() } if oride.DBType != "" && oride.DBType == col.DataType && oride.Nullable != (col.NotNull || col.IsArray) { @@ -284,7 +285,7 @@ func buildModels(r *compiler.Result, settings config.CombinedSettings) []Struct structName = inflection.Singular(structName) } s := Struct{ - Table: core.FQN{Schema: schema.Name, Rel: table.Rel.Name}, + Table: core.FQN{Schema: glob.MustCompile(schema.Name), Rel: glob.MustCompile(table.Rel.Name)}, Name: ModelName(structName, settings), Comment: table.Comment, } @@ -363,7 +364,7 @@ func sameTableName(n *ast.TableName, f core.FQN, defaultSchema string) bool { if n.Schema == "" { schema = defaultSchema } - return n.Catalog == f.Catalog && schema == f.Schema && n.Name == f.Rel + return n.Catalog == f.Catalog && ((f.Schema != nil && f.Schema.Match(schema)) || schema == "") && ((f.Rel != nil && f.Rel.Match(n.Name)) || n.Name == "") } var postgresPlaceholderRegexp = regexp.MustCompile(`\B\$(\d+)\b`) From f45b93eb02c91e08ccc1a709300ce57b1a1087eb Mon Sep 17 00:00:00 2001 From: "Zephaniah E. Loss-Cutler-Hull" Date: Tue, 14 Sep 2021 18:35:13 -0700 Subject: [PATCH 4/8] Add some test cases for override globs. This modifies the existing overrides_go_types test cases to include tests for override globs. --- .../testdata/overrides_go_types/mysql/go/models.go | 12 ++++++++++++ .../testdata/overrides_go_types/mysql/schema.sql | 12 ++++++++++++ .../testdata/overrides_go_types/mysql/sqlc.json | 4 ++++ .../overrides_go_types/postgresql/pgx/go/models.go | 14 +++++++++++++- .../postgresql/pgx/go/query.sql.go | 2 +- .../overrides_go_types/postgresql/pgx/schema.sql | 10 ++++++++++ .../overrides_go_types/postgresql/pgx/sqlc.json | 5 +++-- 7 files changed, 55 insertions(+), 4 deletions(-) diff --git a/internal/endtoend/testdata/overrides_go_types/mysql/go/models.go b/internal/endtoend/testdata/overrides_go_types/mysql/go/models.go index 9b987955d3..28c4e9ecf2 100644 --- a/internal/endtoend/testdata/overrides_go_types/mysql/go/models.go +++ b/internal/endtoend/testdata/overrides_go_types/mysql/go/models.go @@ -6,6 +6,18 @@ import ( "github.com/kyleconroy/sqlc-testdata/pkg" ) +type Bar struct { + Other string + Total int64 + AlsoRetyped pkg.CustomType +} + +type Baz struct { + Other string + Total int64 + AlsoRetyped pkg.CustomType +} + type Foo struct { Other string Total int64 diff --git a/internal/endtoend/testdata/overrides_go_types/mysql/schema.sql b/internal/endtoend/testdata/overrides_go_types/mysql/schema.sql index c0c5fc47dc..31b183ad86 100644 --- a/internal/endtoend/testdata/overrides_go_types/mysql/schema.sql +++ b/internal/endtoend/testdata/overrides_go_types/mysql/schema.sql @@ -3,3 +3,15 @@ CREATE TABLE foo ( total bigint NOT NULL, retyped text NOT NULL ); + +CREATE TABLE bar ( + other text NOT NULL, + total bigint NOT NULL, + also_retyped text NOT NULL +); + +CREATE TABLE baz ( + other text NOT NULL, + total bigint NOT NULL, + also_retyped text NOT NULL +); diff --git a/internal/endtoend/testdata/overrides_go_types/mysql/sqlc.json b/internal/endtoend/testdata/overrides_go_types/mysql/sqlc.json index 592fb072a0..787bf1459f 100644 --- a/internal/endtoend/testdata/overrides_go_types/mysql/sqlc.json +++ b/internal/endtoend/testdata/overrides_go_types/mysql/sqlc.json @@ -11,6 +11,10 @@ { "go_type": "github.com/kyleconroy/sqlc-testdata/pkg.CustomType", "column": "foo.retyped" + }, + { + "go_type": "github.com/kyleconroy/sqlc-testdata/pkg.CustomType", + "column": "*.also_retyped" } ] } diff --git a/internal/endtoend/testdata/overrides_go_types/postgresql/pgx/go/models.go b/internal/endtoend/testdata/overrides_go_types/postgresql/pgx/go/models.go index 6a07b893c1..8edb1fad2d 100644 --- a/internal/endtoend/testdata/overrides_go_types/postgresql/pgx/go/models.go +++ b/internal/endtoend/testdata/overrides_go_types/postgresql/pgx/go/models.go @@ -3,13 +3,25 @@ package override import ( + "database/sql" + orm "database/sql" - "github.com/gofrs/uuid" fuid "github.com/gofrs/uuid" + uuid "github.com/gofrs/uuid" null "github.com/volatiletech/null/v8" null_v4 "gopkg.in/guregu/null.v4" ) +type Bar struct { + ID uuid.UUID + OtherID fuid.UUID + MoreID fuid.UUID + Age sql.NullInt32 + Balance interface{} + Bio sql.NullString + About sql.NullString +} + type Foo struct { ID uuid.UUID OtherID fuid.UUID diff --git a/internal/endtoend/testdata/overrides_go_types/postgresql/pgx/go/query.sql.go b/internal/endtoend/testdata/overrides_go_types/postgresql/pgx/go/query.sql.go index 84a15c92cc..b02ed88fa4 100644 --- a/internal/endtoend/testdata/overrides_go_types/postgresql/pgx/go/query.sql.go +++ b/internal/endtoend/testdata/overrides_go_types/postgresql/pgx/go/query.sql.go @@ -6,7 +6,7 @@ package override import ( "context" - "github.com/gofrs/uuid" + uuid "github.com/gofrs/uuid" ) const loadFoo = `-- name: LoadFoo :many diff --git a/internal/endtoend/testdata/overrides_go_types/postgresql/pgx/schema.sql b/internal/endtoend/testdata/overrides_go_types/postgresql/pgx/schema.sql index 4e0d1f5af7..04551110f3 100644 --- a/internal/endtoend/testdata/overrides_go_types/postgresql/pgx/schema.sql +++ b/internal/endtoend/testdata/overrides_go_types/postgresql/pgx/schema.sql @@ -6,3 +6,13 @@ CREATE TABLE foo ( bio text, about text ); + +CREATE TABLE bar ( + id uuid NOT NULL, + other_id uuid NOT NULL, + more_id uuid NOT NULL, + age integer, + balance double, + bio text, + about text +); diff --git a/internal/endtoend/testdata/overrides_go_types/postgresql/pgx/sqlc.json b/internal/endtoend/testdata/overrides_go_types/postgresql/pgx/sqlc.json index 7290df6fcd..238b937a36 100644 --- a/internal/endtoend/testdata/overrides_go_types/postgresql/pgx/sqlc.json +++ b/internal/endtoend/testdata/overrides_go_types/postgresql/pgx/sqlc.json @@ -10,14 +10,15 @@ "queries": "query.sql", "overrides": [ { - "column": "foo.id", + "column": "*.id", "go_type": { "import": "github.com/gofrs/uuid", + "package": "uuid", "type": "UUID" }, }, { - "column": "foo.other_id", + "column": "*.*_id", "go_type": { "import": "github.com/gofrs/uuid", "package": "fuid", From 7509298e124272dcfdcd16584f041e9760131ec4 Mon Sep 17 00:00:00 2001 From: "Zephaniah E. Loss-Cutler-Hull" Date: Wed, 15 Sep 2021 15:04:48 -0700 Subject: [PATCH 5/8] Move type FQN back to all strings. No more glob.Glob in the FQN. We're isolating the matching in a different place. --- internal/codegen/golang/compat.go | 2 +- internal/codegen/golang/result.go | 4 +--- internal/codegen/kotlin/gen.go | 6 ++---- internal/codegen/python/gen.go | 5 ++--- internal/core/fqn.go | 6 ++---- 5 files changed, 8 insertions(+), 15 deletions(-) diff --git a/internal/codegen/golang/compat.go b/internal/codegen/golang/compat.go index aa38ea3245..cdbb5aac2a 100644 --- a/internal/codegen/golang/compat.go +++ b/internal/codegen/golang/compat.go @@ -13,5 +13,5 @@ func sameTableName(n *ast.TableName, f core.FQN, defaultSchema string) bool { if n.Schema == "" { schema = defaultSchema } - return n.Catalog == f.Catalog && ((f.Schema != nil && f.Schema.Match(schema)) || schema == "") && ((f.Rel != nil && f.Rel.Match(n.Name)) || n.Name == "") + return n.Catalog == f.Catalog && schema == f.Schema && n.Name == f.Rel } diff --git a/internal/codegen/golang/result.go b/internal/codegen/golang/result.go index ed3b9c027f..b207f92f58 100644 --- a/internal/codegen/golang/result.go +++ b/internal/codegen/golang/result.go @@ -5,8 +5,6 @@ import ( "sort" "strings" - "github.com/gobwas/glob" - "github.com/kyleconroy/sqlc/internal/codegen" "github.com/kyleconroy/sqlc/internal/compiler" "github.com/kyleconroy/sqlc/internal/config" @@ -76,7 +74,7 @@ func buildStructs(r *compiler.Result, settings config.CombinedSettings) []Struct structName = inflection.Singular(structName) } s := Struct{ - Table: core.FQN{Schema: glob.MustCompile(schema.Name), Rel: glob.MustCompile(table.Rel.Name)}, + Table: core.FQN{Schema: schema.Name, Rel: table.Rel.Name}, Name: StructName(structName, settings), Comment: table.Comment, } diff --git a/internal/codegen/kotlin/gen.go b/internal/codegen/kotlin/gen.go index 28383b3c9c..48c23f7504 100644 --- a/internal/codegen/kotlin/gen.go +++ b/internal/codegen/kotlin/gen.go @@ -9,8 +9,6 @@ import ( "strings" "text/template" - "github.com/gobwas/glob" - "github.com/kyleconroy/sqlc/internal/codegen" "github.com/kyleconroy/sqlc/internal/compiler" "github.com/kyleconroy/sqlc/internal/config" @@ -28,7 +26,7 @@ func sameTableName(n *ast.TableName, f core.FQN) bool { if n.Schema == "" { schema = "public" } - return n.Catalog == f.Catalog && ((f.Schema != nil && f.Schema.Match(schema)) || schema == "") && ((f.Rel != nil && f.Rel.Match(n.Name)) || n.Name == "") + return n.Catalog == n.Catalog && schema == f.Schema && n.Name == f.Rel } var ktIdentPattern = regexp.MustCompile("[^a-zA-Z0-9_]+") @@ -298,7 +296,7 @@ func buildDataClasses(r *compiler.Result, settings config.CombinedSettings) []St structName = inflection.Singular(structName) } s := Struct{ - Table: core.FQN{Schema: glob.MustCompile(schema.Name), Rel: glob.MustCompile(table.Rel.Name)}, + Table: core.FQN{Schema: schema.Name, Rel: table.Rel.Name}, Name: structName, Comment: table.Comment, } diff --git a/internal/codegen/python/gen.go b/internal/codegen/python/gen.go index 7a145d5718..d0d4cec01c 100644 --- a/internal/codegen/python/gen.go +++ b/internal/codegen/python/gen.go @@ -4,7 +4,6 @@ import ( "bufio" "bytes" "fmt" - "github.com/gobwas/glob" "github.com/kyleconroy/sqlc/internal/codegen" "github.com/kyleconroy/sqlc/internal/compiler" "github.com/kyleconroy/sqlc/internal/config" @@ -285,7 +284,7 @@ func buildModels(r *compiler.Result, settings config.CombinedSettings) []Struct structName = inflection.Singular(structName) } s := Struct{ - Table: core.FQN{Schema: glob.MustCompile(schema.Name), Rel: glob.MustCompile(table.Rel.Name)}, + Table: core.FQN{Schema: schema.Name, Rel: table.Rel.Name}, Name: ModelName(structName, settings), Comment: table.Comment, } @@ -364,7 +363,7 @@ func sameTableName(n *ast.TableName, f core.FQN, defaultSchema string) bool { if n.Schema == "" { schema = defaultSchema } - return n.Catalog == f.Catalog && ((f.Schema != nil && f.Schema.Match(schema)) || schema == "") && ((f.Rel != nil && f.Rel.Match(n.Name)) || n.Name == "") + return n.Catalog == f.Catalog && schema == f.Schema && n.Name == f.Rel } var postgresPlaceholderRegexp = regexp.MustCompile(`\B\$(\d+)\b`) diff --git a/internal/core/fqn.go b/internal/core/fqn.go index a23402fffb..ebdcba8c11 100644 --- a/internal/core/fqn.go +++ b/internal/core/fqn.go @@ -1,11 +1,9 @@ package core -import "github.com/gobwas/glob" - // TODO: This is the last struct left over from the old architecture. Figure // out how to remove it at some point type FQN struct { Catalog string - Schema glob.Glob - Rel glob.Glob + Schema string + Rel string } From d4e91a16e10efc6679f8406e2a4becbf247bbbb8 Mon Sep 17 00:00:00 2001 From: "Zephaniah E. Loss-Cutler-Hull" Date: Wed, 15 Sep 2021 15:05:46 -0700 Subject: [PATCH 6/8] Add some code for doing basic matching. This supports the following: * matches anything zero or more times. ? matches a single character. \* matches a literal *. \? matches a literal ?. Invalid escapes will cause an error. --- internal/config/match.go | 57 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 57 insertions(+) create mode 100644 internal/config/match.go diff --git a/internal/config/match.go b/internal/config/match.go new file mode 100644 index 0000000000..b0f4d534f3 --- /dev/null +++ b/internal/config/match.go @@ -0,0 +1,57 @@ +package config + +import ( + "fmt" + "regexp" +) + +// Match is a wrapper of *regexp.Regexp. +// It contains the match pattern compiled into a regular expression. +type Match struct { + *regexp.Regexp +} + +// Compile takes our match expression as a string, and compiles it into a *Match object. +// Will return an error on an invalid pattern. +func MatchCompile(pattern string) (match *Match, err error) { + regex := "" + escaped := false + arr := []byte(pattern) + + for i := 0; i < len(arr); i++ { + if escaped { + escaped = false + switch arr[i] { + case '*', '?', '\\': + regex += "\\" + string(arr[i]) + default: + return nil, fmt.Errorf("Invalid escaped character '%c'", arr[i]) + } + } else { + switch arr[i] { + case '\\': + escaped = true + case '*': + regex += ".*" + case '?': + regex += "." + case '.', '(', ')', '+', '|', '^', '$', '[', ']', '{', '}': + regex += "\\" + string(arr[i]) + default: + regex += string(arr[i]) + } + } + } + + if escaped { + return nil, fmt.Errorf("Unterminated escape at end of pattern") + } + + var r *regexp.Regexp + + if r, err = regexp.Compile("^" + regex + "$"); err != nil { + return nil, err + } + + return &Match{r}, nil +} From d5d9454d02407cd99ec148bac8b24cf64a4b5367 Mon Sep 17 00:00:00 2001 From: "Zephaniah E. Loss-Cutler-Hull" Date: Wed, 15 Sep 2021 15:07:25 -0700 Subject: [PATCH 7/8] Rework overrides, no FQN, internal Match. So the Override struct no longer uses a FQN, and instead has split out *Match items for ColumnName, TableCatalog, TableSchema, and TableRel. There is a new *Override.Matches function that indicates if a given *ast.TableName (with a default schema argument) matches the override. And the code that used to use sameTableName in the python and golang code generators now uses the Matches function. --- internal/codegen/golang/go_type.go | 4 +- internal/codegen/python/gen.go | 4 +- internal/config/config.go | 82 +++++++++++++++++++++++++----- 3 files changed, 74 insertions(+), 16 deletions(-) diff --git a/internal/codegen/golang/go_type.go b/internal/codegen/golang/go_type.go index d678b988a5..718a8dca1c 100644 --- a/internal/codegen/golang/go_type.go +++ b/internal/codegen/golang/go_type.go @@ -11,8 +11,8 @@ func goType(r *compiler.Result, col *compiler.Column, settings config.CombinedSe if oride.GoTypeName == "" { continue } - sameTable := sameTableName(col.Table, oride.Table, r.Catalog.DefaultSchema) - if oride.Column != "" && oride.ColumnName.Match(col.Name) && sameTable { + sameTable := oride.Matches(col.Table, r.Catalog.DefaultSchema) + if oride.Column != "" && oride.ColumnName.MatchString(col.Name) && sameTable { return oride.GoTypeName } } diff --git a/internal/codegen/python/gen.go b/internal/codegen/python/gen.go index d0d4cec01c..3173bc6d8f 100644 --- a/internal/codegen/python/gen.go +++ b/internal/codegen/python/gen.go @@ -181,8 +181,8 @@ func pyInnerType(r *compiler.Result, col *compiler.Column, settings config.Combi if !oride.PythonType.IsSet() { continue } - sameTable := sameTableName(col.Table, oride.Table, r.Catalog.DefaultSchema) - if oride.Column != "" && oride.ColumnName.Match(col.Name) && sameTable { + sameTable := oride.Matches(col.Table, r.Catalog.DefaultSchema) + if oride.Column != "" && oride.ColumnName.MatchString(col.Name) && sameTable { return oride.PythonType.TypeString() } if oride.DBType != "" && oride.DBType == col.DataType && oride.Nullable != (col.NotNull || col.IsArray) { diff --git a/internal/config/config.go b/internal/config/config.go index f17d8d97fc..90f41da815 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -9,10 +9,9 @@ import ( "os" "strings" - "github.com/kyleconroy/sqlc/internal/core" - yaml "gopkg.in/yaml.v3" + "github.com/kyleconroy/sqlc/internal/sql/ast" - "github.com/gobwas/glob" + yaml "gopkg.in/yaml.v3" ) const errMessageNoVersion = `The configuration file must have a version number. @@ -167,15 +166,50 @@ type Override struct { // fully qualified name of the column, e.g. `accounts.id` Column string `json:"column" yaml:"column"` - ColumnName glob.Glob - Table core.FQN + ColumnName *Match + TableCatalog *Match + TableSchema *Match + TableRel *Match GoImportPath string GoPackage string GoTypeName string GoBasicType bool } -func (o *Override) Parse() error { +func (o *Override) Matches(n *ast.TableName, defaultSchema string) bool { + if n == nil { + return false + } + + schema := n.Schema + if n.Schema == "" { + schema = defaultSchema + } + + if o.TableCatalog != nil && !o.TableCatalog.MatchString(n.Catalog) { + return false + } + + if o.TableSchema == nil && schema != "" { + return false + } + + if o.TableSchema != nil && !o.TableSchema.MatchString(schema) { + return false + } + + if o.TableRel == nil && n.Name != "" { + return false + } + + if o.TableRel != nil && !o.TableRel.MatchString(n.Name) { + return false + } + + return true +} + +func (o *Override) Parse() (err error) { // validate deprecated postgres_type field if o.Deprecated_PostgresType != "" { @@ -205,14 +239,38 @@ func (o *Override) Parse() error { colParts := strings.Split(o.Column, ".") switch len(colParts) { case 2: - o.ColumnName = glob.MustCompile(colParts[1]) - o.Table = core.FQN{Schema: glob.MustCompile("public"), Rel: glob.MustCompile(colParts[0])} + if o.ColumnName, err = MatchCompile(colParts[1]); err != nil { + return err + } + if o.TableRel, err = MatchCompile(colParts[0]); err != nil { + return err + } + if o.TableSchema, err = MatchCompile("public"); err != nil { + return err + } case 3: - o.ColumnName = glob.MustCompile(colParts[2]) - o.Table = core.FQN{Schema: glob.MustCompile(colParts[0]), Rel: glob.MustCompile(colParts[1])} + if o.ColumnName, err = MatchCompile(colParts[2]); err != nil { + return err + } + if o.TableRel, err = MatchCompile(colParts[1]); err != nil { + return err + } + if o.TableSchema, err = MatchCompile(colParts[0]); err != nil { + return err + } case 4: - o.ColumnName = glob.MustCompile(colParts[3]) - o.Table = core.FQN{Catalog: colParts[0], Schema: glob.MustCompile(colParts[1]), Rel: glob.MustCompile(colParts[2])} + if o.ColumnName, err = MatchCompile(colParts[3]); err != nil { + return err + } + if o.TableRel, err = MatchCompile(colParts[2]); err != nil { + return err + } + if o.TableSchema, err = MatchCompile(colParts[1]); err != nil { + return err + } + if o.TableCatalog, err = MatchCompile(colParts[0]); err != nil { + return err + } default: return fmt.Errorf("Override `column` specifier %q is not the proper format, expected '[catalog.][schema.]tablename.colname'", o.Column) } From a0c9b45c4a5a40e51ddd1aaade4a523d76dc9e8b Mon Sep 17 00:00:00 2001 From: "Zephaniah E. Loss-Cutler-Hull" Date: Wed, 15 Sep 2021 15:10:11 -0700 Subject: [PATCH 8/8] Drop the last traces of the glob library. We're doing our own matching thing now. --- go.mod | 1 - go.sum | 2 -- 2 files changed, 3 deletions(-) diff --git a/go.mod b/go.mod index b8b16625d8..5bece99fa2 100644 --- a/go.mod +++ b/go.mod @@ -6,7 +6,6 @@ require ( github.com/antlr/antlr4 v0.0.0-20200209180723-1177c0b58d07 github.com/davecgh/go-spew v1.1.1 github.com/go-sql-driver/mysql v1.6.0 - github.com/gobwas/glob v0.2.3 github.com/google/go-cmp v0.5.6 github.com/jackc/pgx/v4 v4.13.0 github.com/jinzhu/inflection v1.0.0 diff --git a/go.sum b/go.sum index ee03990eb1..33add28121 100644 --- a/go.sum +++ b/go.sum @@ -93,8 +93,6 @@ github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LB github.com/go-sql-driver/mysql v1.6.0 h1:BCTh4TKNUYmOmMUcQ3IipzF5prigylS7XXjEkfCHuOE= github.com/go-sql-driver/mysql v1.6.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= -github.com/gobwas/glob v0.2.3 h1:A4xDbljILXROh+kObIiy5kIaPYD8e96x1tgBhUI5J+Y= -github.com/gobwas/glob v0.2.3/go.mod h1:d3Ez4x06l9bZtSvzIay5+Yzi0fmZzPgnTbPcKjJAkT8= github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/gofrs/uuid v4.0.0+incompatible h1:1SD/1F5pU8p29ybwgQSwpQk+mwdRrXCYuPhW6m+TnJw= github.com/gofrs/uuid v4.0.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM=