add support for custom Go struct tags (#1569)

* internal/config: use strings.Trim{Prefix,Suffix}

This is equivalent and slightly simpler.

* Makefile: fix vtproto 'go install' command

* internal/codegen/golang: simplify template tag condition

Rather than modeling when .Tag will be empty,
check directly whether .Tag is empty.

This simplifies the template and reduces the number
of places that must be touched when adding new
sources of struct tags.

* internal/codegen/golang: tweak tag formatting

Rather than inserting the colon at tag construction time,
insert it at tag formatting time.

This makes the input look a bit more natural.
This matters more, as we are about to add another,
more distant, place where we insert tags.

* all: add support for custom Go struct tags

This change adds a new type of override: go_struct_tag.
When provided for a field, it adds that struct tag to the generated code.
The provided struct tag is parsed according to the standard
package reflect rules, and its components are updated independently.
This allows struct tag overrides to be compatible with (and optionally
override) autogenerated json and db struct tags.

Fixes #534

* go.mod: bump to Go 1.18

The code uses some 1.18-only features, like strings.Cut and testing.F.

The CI requires Go 1.18.

Since Go 1.18 is now required, reflect that in the go.mod.

Author: josharian

Makefile

@@ -38,7 +38,7 @@ mysqlsh:
 # $ protoc --version
 # libprotoc 3.19.1
 # $ go install google.golang.org/protobuf/cmd/protoc-gen-go@latest
-# $ go install github.com/planetscale/vtprotobuf/cmd/protoc-gen-go-vtproto
+# $ go install github.com/planetscale/vtprotobuf/cmd/protoc-gen-go-vtproto@latest
 proto: internal/plugin/codegen.pb.go internal/python/ast/ast.pb.go
 
 internal/plugin/codegen.pb.go: protos/plugin/codegen.proto

docs/howto/structs.md

@@ -46,3 +46,7 @@ type Author struct {
 	CreatedAt time.Time `json:"created_at"`
 }
 ```
+
+## More control
+
+See the Type Overrides section of the Configuration File docs for fine-grained control over struct field types and tags.

docs/reference/config.md

@@ -225,6 +225,9 @@ Each override document has the following keys:
   - The PostgreSQL or MySQL type to override. Find the full list of supported types in [postgresql_type.go](https://github.com/kyleconroy/sqlc/blob/main/internal/codegen/golang/postgresql_type.go#L12) or [mysql_type.go](https://github.com/kyleconroy/sqlc/blob/main/internal/codegen/golang/mysql_type.go#L12). Note that for Postgres you must use the pg_catalog prefixed names where available.
 - `go_type`:
   - A fully qualified name to a Go type to use in the generated code.
+- `go_struct_tag`:
+  - A reflect-style struct tag to use in the generated code, e.g. `a:"b" x:"y,z"`.
+    If you want general json/db tags for all fields, use `emit_db_tags` and/or `emit_json_tags` instead.
 - `nullable`:
   - If true, use this type when a column is nullable. Defaults to `false`.
 

go.mod

@@ -1,6 +1,6 @@
 module github.com/kyleconroy/sqlc
 
-go 1.17
+go 1.18
 
 require (
 	github.com/antlr/antlr4/runtime/Go/antlr v0.0.0-20220209173558-ad29539cd2e9

internal/cmd/shim.go

@@ -109,6 +109,7 @@ func pluginGoType(o config.Override) *plugin.ParsedGoType {
 		Package:    o.GoPackage,
 		TypeName:   o.GoTypeName,
 		BasicType:  o.GoBasicType,
+		StructTags: o.GoStructTags,
 	}
 }
 

internal/codegen/golang/field.go

@@ -19,7 +19,7 @@ type Field struct {
 func (gf Field) Tag() string {
 	tags := make([]string, 0, len(gf.Tags))
 	for key, val := range gf.Tags {
-		tags = append(tags, fmt.Sprintf("%s\"%s\"", key, val))
+		tags = append(tags, fmt.Sprintf("%s:\"%s\"", key, val))
 	}
 	if len(tags) == 0 {
 		return ""

internal/codegen/golang/go_type.go

@@ -5,6 +5,26 @@ import (
 	"github.com/kyleconroy/sqlc/internal/plugin"
 )
 
+func addExtraGoStructTags(tags map[string]string, req *plugin.CodeGenRequest, col *plugin.Column) {
+	for _, oride := range req.Settings.Overrides {
+		if oride.GoType.StructTags == nil {
+			continue
+		}
+		if !sdk.Matches(oride, col.Table, req.Catalog.DefaultSchema) {
+			// Different table.
+			continue
+		}
+		if !sdk.MatchString(oride.ColumnName, col.Name) {
+			// Different column.
+			continue
+		}
+		// Add the extra tags.
+		for k, v := range oride.GoType.StructTags {
+			tags[k] = v
+		}
+	}
+}
+
 func goType(req *plugin.CodeGenRequest, col *plugin.Column) string {
 	// Check if the column's type has been overridden
 	for _, oride := range req.Settings.Overrides {

internal/codegen/golang/result.go

@@ -74,11 +74,12 @@ func buildStructs(req *plugin.CodeGenRequest) []Struct {
 			for _, column := range table.Columns {
 				tags := map[string]string{}
 				if req.Settings.Go.EmitDbTags {
-					tags["db:"] = column.Name
+					tags["db"] = column.Name
 				}
 				if req.Settings.Go.EmitJsonTags {
-					tags["json:"] = JSONTagName(column.Name, req.Settings)
+					tags["json"] = JSONTagName(column.Name, req.Settings)
 				}
+				addExtraGoStructTags(tags, req, column)
 				s.Fields = append(s.Fields, Field{
 					Name:    StructName(column.Name, req.Settings),
 					Type:    goType(req, column),
@@ -283,10 +284,10 @@ func columnsToStruct(req *plugin.CodeGenRequest, name string, columns []goColumn
 		}
 		tags := map[string]string{}
 		if req.Settings.Go.EmitDbTags {
-			tags["db:"] = tagName
+			tags["db"] = tagName
 		}
 		if req.Settings.Go.EmitJsonTags {
-			tags["json:"] = JSONTagName(tagName, req.Settings)
+			tags["json"] = JSONTagName(tagName, req.Settings)
 		}
 		gs.Fields = append(gs.Fields, Field{
 			Name:   fieldName,

internal/codegen/golang/templates/pgx/batchCode.tmpl

@@ -12,14 +12,14 @@ type {{.MethodName}}BatchResults struct {
 
 {{if .Arg.EmitStruct}}
 type {{.Arg.Type}} struct { {{- range .Arg.Struct.Fields}}
-  {{.Name}} {{.Type}} {{if or ($.EmitJSONTags) ($.EmitDBTags)}}{{$.Q}}{{.Tag}}{{$.Q}}{{end}}
+  {{.Name}} {{.Type}} {{if .Tag}}{{$.Q}}{{.Tag}}{{$.Q}}{{end}}
   {{- end}}
 }
 {{end}}
 
 {{if .Ret.EmitStruct}}
 type {{.Ret.Type}} struct { {{- range .Ret.Struct.Fields}}
-  {{.Name}} {{.Type}} {{if or ($.EmitJSONTags) ($.EmitDBTags)}}{{$.Q}}{{.Tag}}{{$.Q}}{{end}}
+  {{.Name}} {{.Type}} {{if .Tag}}{{$.Q}}{{.Tag}}{{$.Q}}{{end}}
   {{- end}}
 }
 {{end}}

internal/codegen/golang/templates/pgx/queryCode.tmpl

@@ -10,14 +10,14 @@ const {{.ConstantName}} = {{$.Q}}-- name: {{.MethodName}} {{.Cmd}}
 {{if ne (hasPrefix .Cmd ":batch") true}}
 {{if .Arg.EmitStruct}}
 type {{.Arg.Type}} struct { {{- range .Arg.Struct.Fields}}
-  {{.Name}} {{.Type}} {{if or ($.EmitJSONTags) ($.EmitDBTags)}}{{$.Q}}{{.Tag}}{{$.Q}}{{end}}
+  {{.Name}} {{.Type}} {{if .Tag}}{{$.Q}}{{.Tag}}{{$.Q}}{{end}}
   {{- end}}
 }
 {{end}}
 
 {{if .Ret.EmitStruct}}
 type {{.Ret.Type}} struct { {{- range .Ret.Struct.Fields}}
-  {{.Name}} {{.Type}} {{if or ($.EmitJSONTags) ($.EmitDBTags)}}{{$.Q}}{{.Tag}}{{$.Q}}{{end}}
+  {{.Name}} {{.Type}} {{if .Tag}}{{$.Q}}{{.Tag}}{{$.Q}}{{end}}
   {{- end}}
 }
 {{end}}

internal/codegen/golang/templates/stdlib/queryCode.tmpl

@@ -7,14 +7,14 @@ const {{.ConstantName}} = {{$.Q}}-- name: {{.MethodName}} {{.Cmd}}
 
 {{if .Arg.EmitStruct}}
 type {{.Arg.Type}} struct { {{- range .Arg.UniqueFields}}
-  {{.Name}} {{.Type}} {{if or ($.EmitJSONTags) ($.EmitDBTags)}}{{$.Q}}{{.Tag}}{{$.Q}}{{end}}
+  {{.Name}} {{.Type}} {{if .Tag}}{{$.Q}}{{.Tag}}{{$.Q}}{{end}}
   {{- end}}
 }
 {{end}}
 
 {{if .Ret.EmitStruct}}
 type {{.Ret.Type}} struct { {{- range .Ret.Struct.Fields}}
-  {{.Name}} {{.Type}} {{if or ($.EmitJSONTags) ($.EmitDBTags)}}{{$.Q}}{{.Tag}}{{$.Q}}{{end}}
+  {{.Name}} {{.Type}} {{if .Tag}}{{$.Q}}{{.Tag}}{{$.Q}}{{end}}
   {{- end}}
 }
 {{end}}

internal/codegen/golang/templates/template.tmpl

@@ -111,7 +111,7 @@ type {{.Name}} struct { {{- range .Fields}}
   {{- if .Comment}}
   {{comment .Comment}}{{else}}
   {{- end}}
-  {{.Name}} {{.Type}} {{if or ($.EmitJSONTags) ($.EmitDBTags)}}{{$.Q}}{{.Tag}}{{$.Q}}{{end}}
+  {{.Name}} {{.Type}} {{if .Tag}}{{$.Q}}{{.Tag}}{{$.Q}}{{end}}
   {{- end}}
 }
 {{end}}

internal/config/config.go

@@ -167,6 +167,10 @@ type Override struct {
 	// name of the golang type to use, e.g. `github.com/segmentio/ksuid.KSUID`
 	GoType GoType `json:"go_type" yaml:"go_type"`
 
+	// additional Go struct tags to add to this field, in raw Go struct tag form, e.g. `validate:"required" x:"y,z"`
+	// see https://github.com/kyleconroy/sqlc/issues/534
+	GoStructTag GoStructTag `json:"go_struct_tag" yaml:"go_struct_tag"`
+
 	// name of the python type to use, e.g. `mymodule.TypeName`
 	PythonType PythonType `json:"python_type" yaml:"python_type"`
 
@@ -193,6 +197,9 @@ type Override struct {
 	GoPackage    string
 	GoTypeName   string
 	GoBasicType  bool
+
+	// Parsed form of GoStructTag, e.g. {"validate:", "required"}
+	GoStructTags map[string]string
 }
 
 func (o *Override) Matches(n *ast.TableName, defaultSchema string) bool {
@@ -305,6 +312,13 @@ func (o *Override) Parse() (err error) {
 	o.GoTypeName = parsed.TypeName
 	o.GoBasicType = parsed.BasicType
 
+	// validate GoStructTag
+	tags, err := o.GoStructTag.Parse()
+	if err != nil {
+		return err
+	}
+	o.GoStructTags = tags
+
 	return nil
 }
 

internal/config/go_type.go

@@ -22,6 +22,7 @@ type ParsedGoType struct {
 	Package    string
 	TypeName   string
 	BasicType  bool
+	StructTag  string
 }
 
 func (o *GoType) UnmarshalJSON(data []byte) error {
@@ -138,16 +139,12 @@ func (gt GoType) Parse() (*ParsedGoType, error) {
 			return nil, fmt.Errorf("Package override `go_type` specifier %q is not the proper format, expected 'package.type', e.g. 'github.com/segmentio/ksuid.KSUID'", input)
 		}
 		typename = input[lastSlash+1:]
-		if strings.HasPrefix(typename, "go-") {
-			// a package name beginning with "go-" will give syntax errors in
-			// generated code. We should do the right thing and get the actual
-			// import name, but in lieu of that, stripping the leading "go-" may get
-			// us what we want.
-			typename = typename[len("go-"):]
-		}
-		if strings.HasSuffix(typename, "-go") {
-			typename = typename[:len(typename)-len("-go")]
-		}
+		// a package name beginning with "go-" will give syntax errors in
+		// generated code. We should do the right thing and get the actual
+		// import name, but in lieu of that, stripping the leading "go-" may get
+		// us what we want.
+		typename = strings.TrimPrefix(typename, "go-")
+		typename = strings.TrimSuffix(typename, "-go")
 		o.ImportPath = input[:lastDot]
 	}
 	o.TypeName = typename
@@ -158,3 +155,31 @@ func (gt GoType) Parse() (*ParsedGoType, error) {
 	}
 	return &o, nil
 }
+
+// GoStructTag is a raw Go struct tag.
+type GoStructTag string
+
+// Parse parses and validates a GoStructTag.
+// The output is in a form convenient for codegen.
+//
+// Sample valid inputs/outputs:
+//
+// In                    Out
+// empty string      {}
+// `a:"b"`           {"a": "b"}
+// `a:"b" x:"y,z"`   {"a": "b", "x": "y,z"}
+func (s GoStructTag) Parse() (map[string]string, error) {
+	m := make(map[string]string)
+	fields := strings.Fields(string(s))
+	for _, f := range fields {
+		k, v, ok := strings.Cut(f, ":")
+		if !ok {
+			return nil, fmt.Errorf("Failed to parse Go struct tag: no colon in field %q", f)
+		}
+		if len(v) < 2 || v[0] != '"' || v[len(v)-1] != '"' {
+			return nil, fmt.Errorf("Failed to parse Go struct tag: missing quotes around value in field %q", f)
+		}
+		m[k] = v[1 : len(v)-1] // trim quotes off of v
+	}
+	return m, nil
+}

internal/endtoend/testdata/overrides_go_struct_tags/invalid_tags/query.sql

@@ -0,0 +1 @@
+SELECT 1;

internal/endtoend/testdata/overrides_go_struct_tags/invalid_tags/schema.sql

@@ -0,0 +1,14 @@
+CREATE TABLE foo (
+    other     text NOT NULL,
+    tagged    text NOT NULL
+);
+
+CREATE TABLE bar (
+    other          text NOT NULL,
+    also_tagged    text NOT NULL
+);
+
+CREATE TABLE baz (
+    other          text NOT NULL,
+    also_tagged    text NOT NULL
+);

internal/endtoend/testdata/overrides_go_struct_tags/invalid_tags/sqlc.json

@@ -0,0 +1,22 @@
+{
+  "version": "1",
+  "packages": [
+    {
+      "path": "go",
+      "name": "override",
+      "engine": "mysql",
+      "schema": "schema.sql",
+      "queries": "query.sql",
+      "overrides": [
+        {
+          "go_struct_tag": "abc",
+          "column": "foo.tagged"
+        },
+        {
+          "go_struct_tag": "a:b",
+          "column": "*.also_tagged"
+        }
+      ]
+    }
+  ]
+}

internal/endtoend/testdata/overrides_go_struct_tags/invalid_tags/stderr.txt

@@ -0,0 +1 @@
+error parsing sqlc.json: Failed to parse Go struct tag: no colon in field "abc"

internal/endtoend/testdata/overrides_go_struct_tags/mysql/go/db.go

@@ -0,0 +1 @@
+SELECT 1;

internal/endtoend/testdata/overrides_go_struct_tags/mysql/go/models.go

@@ -0,0 +1,14 @@
+CREATE TABLE foo (
+    other     text NOT NULL,
+    tagged    text NOT NULL
+);
+
+CREATE TABLE bar (
+    other          text NOT NULL,
+    also_tagged    text NOT NULL
+);
+
+CREATE TABLE baz (
+    other          text NOT NULL,
+    also_tagged    text NOT NULL
+);