Schemas and queries can now be a list of paths (#426)

Co-authored-by: Kyle Conroy <kyle@conroy.org>

Author: jskelcy

examples/authors/db_test.go

@@ -11,7 +11,7 @@ import (
 )
 
 func TestAuthors(t *testing.T) {
-	sdb, cleanup := sqltest.PostgreSQL(t, "schema.sql")
+	sdb, cleanup := sqltest.PostgreSQL(t, []string{"schema.sql"})
 	defer cleanup()
 
 	ctx := context.Background()

examples/booktest/postgresql/db_test.go

@@ -4,15 +4,14 @@ package booktest
 
 import (
 	"context"
-	"path/filepath"
 	"testing"
 	"time"
 
 	"github.com/kyleconroy/sqlc/internal/sqltest"
 )
 
 func TestBooks(t *testing.T) {
-	db, cleanup := sqltest.PostgreSQL(t, filepath.Join("schema.sql"))
+	db, cleanup := sqltest.PostgreSQL(t, []string{"schema.sql"})
 	defer cleanup()
 
 	ctx := context.Background()

examples/ondeck/db_test.go

@@ -123,7 +123,7 @@ func runOnDeckQueries(t *testing.T, q *Queries) {
 func TestPrepared(t *testing.T) {
 	t.Parallel()
 
-	sdb, cleanup := sqltest.PostgreSQL(t, "schema")
+	sdb, cleanup := sqltest.PostgreSQL(t, []string{"schema"})
 	defer cleanup()
 
 	q, err := Prepare(context.Background(), sdb)
@@ -137,7 +137,7 @@ func TestPrepared(t *testing.T) {
 func TestQueries(t *testing.T) {
 	t.Parallel()
 
-	sdb, cleanup := sqltest.PostgreSQL(t, "schema")
+	sdb, cleanup := sqltest.PostgreSQL(t, []string{"schema"})
 	defer cleanup()
 
 	runOnDeckQueries(t, New(sdb))

go.sum

@@ -257,9 +257,11 @@ github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxv
 github.com/konsorten/go-windows-terminal-sequences v1.0.2 h1:DB17ag19krx9CFsz4o3enTrPXyIXCl+2iCXH/aMAp9s=
 github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
 github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc=
+github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI=
 github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
 github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
 github.com/kr/pty v1.1.8/go.mod h1:O1sed60cT9XZ5uDucP5qwvh+TE3NnUj51EiZO/lmSfw=
+github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
 github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
 github.com/krishicks/yaml-patch v0.0.10/go.mod h1:Sm5TchwZS6sm7RJoyg87tzxm2ZcKzdRE4Q7TjNhPrME=
 github.com/lfittl/pg_query_go v1.0.0 h1:rcHZK5DBEUoxtO6dACP+UVCHKtA1ZsELBW0rSjOXMAE=
@@ -674,6 +676,7 @@ gopkg.in/alecthomas/kingpin.v3-unstable v3.0.0-20180810215634-df19058c872c/go.mo
 gopkg.in/asn1-ber.v1 v1.0.0-20150924051756-4e86f4367175/go.mod h1:cuepJuh7vyXfUyUwEgHQXw849cJrilpS5NeIjOWESAw=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo=
 gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/cheggaaa/pb.v1 v1.0.25/go.mod h1:V/YB90LKu/1FcN3WVnfiiE5oMCibMjukxqG/qStrOgw=
 gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=

internal/cmd/generate.go

@@ -113,8 +113,17 @@ func Generate(dir string, stderr io.Writer) (map[string]string, error) {
 		var result dinosql.Generateable
 
 		// TODO: This feels like a hack that will bite us later
-		sql.Schema = filepath.Join(dir, sql.Schema)
-		sql.Queries = filepath.Join(dir, sql.Queries)
+		joined := make([]string, 0, len(sql.Schema))
+		for _, s := range sql.Schema {
+			joined = append(joined, filepath.Join(dir, s))
+		}
+		sql.Schema = joined
+
+		joined = make([]string, 0, len(sql.Queries))
+		for _, q := range sql.Queries {
+			joined = append(joined, filepath.Join(dir, q))
+		}
+		sql.Queries = joined
 
 		var name string
 		parseOpts := dinosql.ParserOpts{}

internal/compiler/compile.go

@@ -71,10 +71,15 @@ func Run(conf config.SQL, combo config.CombinedSettings) (*Result, error) {
 		return nil, fmt.Errorf("unknown engine: %s", conf.Engine)
 	}
 
-	rd, err := os.Open(conf.Schema)
-	if err != nil {
-		return nil, err
+	blobs := make([]io.Reader, 0, len(conf.Schema))
+	for _, s := range conf.Schema {
+		b, err := os.Open(s)
+		if err != nil {
+			return nil, err
+		}
+		blobs = append(blobs, b)
 	}
+	rd := io.MultiReader(blobs...)
 
 	stmts, err := p.Parse(rd)
 	if err != nil {

internal/config/config.go

@@ -2,6 +2,7 @@ package config
 
 import (
 	"bytes"
+	"encoding/json"
 	"errors"
 	"fmt"
 	"go/types"
@@ -35,6 +36,39 @@ type versionSetting struct {
 
 type Engine string
 
+type Paths []string
+
+func (p *Paths) UnmarshalJSON(data []byte) error {
+	if string(data[0]) == `[` {
+		var out []string
+		if err := json.Unmarshal(data, &out); err != nil {
+			return nil
+		}
+		*p = Paths(out)
+		return nil
+	}
+	var out string
+	if err := json.Unmarshal(data, &out); err != nil {
+		return nil
+	}
+	*p = Paths([]string{out})
+	return nil
+}
+
+func (p *Paths) UnmarshalYAML(unmarshal func(interface{}) error) error {
+	out := []string{}
+	if sliceErr := unmarshal(&out); sliceErr != nil {
+		var ele string
+		if strErr := unmarshal(&ele); strErr != nil {
+			return strErr
+		}
+		out = []string{ele}
+	}
+
+	*p = Paths(out)
+	return nil
+}
+
 const (
 	EngineMySQL      Engine = "mysql"
 	EnginePostgreSQL Engine = "postgresql"
@@ -67,8 +101,8 @@ type GenKotlin struct {
 
 type SQL struct {
 	Engine  Engine `json:"engine,omitempty" yaml:"engine"`
-	Schema  string `json:"schema" yaml:"schema"`
-	Queries string `json:"queries" yaml:"queries"`
+	Schema  Paths  `json:"schema" yaml:"schema"`
+	Queries Paths  `json:"queries" yaml:"queries"`
 	Gen     SQLGen `json:"gen" yaml:"gen"`
 }
 

internal/config/v_one.go

@@ -19,8 +19,8 @@ type v1PackageSettings struct {
 	Name                string     `json:"name" yaml:"name"`
 	Engine              Engine     `json:"engine,omitempty" yaml:"engine"`
 	Path                string     `json:"path" yaml:"path"`
-	Schema              string     `json:"schema" yaml:"schema"`
-	Queries             string     `json:"queries" yaml:"queries"`
+	Schema              Paths      `json:"schema" yaml:"schema"`
+	Queries             Paths      `json:"queries" yaml:"queries"`
 	EmitInterface       bool       `json:"emit_interface" yaml:"emit_interface"`
 	EmitJSONTags        bool       `json:"emit_json_tags" yaml:"emit_json_tags"`
 	EmitPreparedQueries bool       `json:"emit_prepared_queries" yaml:"emit_prepared_queries"`

internal/dinosql/parser.go

@@ -60,43 +60,34 @@ func (e *ParserErr) Error() string {
 	return fmt.Sprintf("multiple errors: %d errors", len(e.Errs))
 }
 
-func ReadSQLFiles(path string) ([]string, error) {
-	f, err := os.Stat(path)
-	if err != nil {
-		return nil, fmt.Errorf("path %s does not exist", path)
-	}
-
+func ReadSQLFiles(paths []string) ([]string, error) {
 	var files []string
-	if f.IsDir() {
-		listing, err := ioutil.ReadDir(path)
+	for _, path := range paths {
+		f, err := os.Stat(path)
 		if err != nil {
-			return nil, err
+			return nil, fmt.Errorf("path %s does not exist", path)
 		}
-		for _, f := range listing {
-			files = append(files, filepath.Join(path, f.Name()))
-		}
-	} else {
-		files = append(files, path)
-	}
 
-	var sql []string
-	for _, filename := range files {
-		if !strings.HasSuffix(filename, ".sql") {
-			continue
-		}
-		if strings.HasPrefix(filepath.Base(filename), ".") {
-			continue
+		if f.IsDir() {
+			listing, err := ioutil.ReadDir(path)
+			if err != nil {
+				return nil, err
+			}
+			for _, f := range listing {
+				files = append(files, filepath.Join(path, f.Name()))
+			}
+		} else {
+			files = append(files, path)
 		}
-		if migrations.IsDown(filename) {
+		if migrations.IsDown(path) {
 			continue
 		}
-		sql = append(sql, filename)
 	}
-	return sql, nil
+	return files, nil
 }
 
-func ParseCatalog(schema string) (core.Catalog, error) {
-	files, err := ReadSQLFiles(schema)
+func ParseCatalog(schemas []string) (core.Catalog, error) {
+	files, err := ReadSQLFiles(schemas)
 	if err != nil {
 		return core.Catalog{}, err
 	}
@@ -197,73 +188,75 @@ type ParserOpts struct {
 	UsePositionalParameters bool
 }
 
-func ParseQueries(c core.Catalog, queries string, opts ParserOpts) (*Result, error) {
-	f, err := os.Stat(queries)
-	if err != nil {
-		return nil, fmt.Errorf("path %s does not exist", queries)
-	}
-
-	var files []string
-	if f.IsDir() {
-		listing, err := ioutil.ReadDir(queries)
-		if err != nil {
-			return nil, err
-		}
-		for _, f := range listing {
-			files = append(files, filepath.Join(queries, f.Name()))
-		}
-	} else {
-		files = append(files, queries)
-	}
-
+func ParseQueries(c core.Catalog, queriesPaths []string, opts ParserOpts) (*Result, error) {
 	merr := NewParserErr()
 	var q []*Query
-	set := map[string]struct{}{}
-	for _, filename := range files {
-		if !strings.HasSuffix(filename, ".sql") {
-			continue
-		}
-		if strings.HasPrefix(filepath.Base(filename), ".") {
-			continue
-		}
-		blob, err := ioutil.ReadFile(filename)
+	for _, queries := range queriesPaths {
+		f, err := os.Stat(queries)
 		if err != nil {
-			merr.Add(filename, "", 0, err)
-			continue
+			return nil, fmt.Errorf("path %s does not exist", queries)
 		}
-		source := string(blob)
-		tree, err := pg.Parse(source)
-		if err != nil {
-			merr.Add(filename, source, 0, err)
-			continue
+
+		var files []string
+		if f.IsDir() {
+			listing, err := ioutil.ReadDir(queries)
+			if err != nil {
+				return nil, err
+			}
+			for _, f := range listing {
+				files = append(files, filepath.Join(queries, f.Name()))
+			}
+		} else {
+			files = append(files, queries)
 		}
-		for _, stmt := range tree.Statements {
-			query, err := parseQuery(c, stmt, source, opts.UsePositionalParameters)
-			if err == errUnsupportedStatementType {
+
+		set := map[string]struct{}{}
+		for _, filename := range files {
+			if !strings.HasSuffix(filename, ".sql") {
+				continue
+			}
+			if strings.HasPrefix(filepath.Base(filename), ".") {
 				continue
 			}
+			blob, err := ioutil.ReadFile(filename)
 			if err != nil {
-				merr.Add(filename, source, location(stmt), err)
+				merr.Add(filename, "", 0, err)
 				continue
 			}
-			if query.Name != "" {
-				if _, exists := set[query.Name]; exists {
-					merr.Add(filename, source, location(stmt), fmt.Errorf("duplicate query name: %s", query.Name))
+			source := string(blob)
+			tree, err := pg.Parse(source)
+			if err != nil {
+				merr.Add(filename, source, 0, err)
+				continue
+			}
+			for _, stmt := range tree.Statements {
+				query, err := parseQuery(c, stmt, source, opts.UsePositionalParameters)
+				if err == errUnsupportedStatementType {
 					continue
 				}
-				set[query.Name] = struct{}{}
-			}
-			query.Filename = filepath.Base(filename)
-			if query != nil {
-				q = append(q, query)
+				if err != nil {
+					merr.Add(filename, source, location(stmt), err)
+					continue
+				}
+				if query.Name != "" {
+					if _, exists := set[query.Name]; exists {
+						merr.Add(filename, source, location(stmt), fmt.Errorf("duplicate query name: %s", query.Name))
+						continue
+					}
+					set[query.Name] = struct{}{}
+				}
+				query.Filename = filepath.Base(filename)
+				if query != nil {
+					q = append(q, query)
+				}
 			}
 		}
 	}
 	if len(merr.Errs) > 0 {
 		return nil, merr
 	}
 	if len(q) == 0 {
-		return nil, fmt.Errorf("path %s contains no queries", queries)
+		return nil, fmt.Errorf("no queries contained in paths %s", strings.Join(queriesPaths, ","))
 	}
 	return &Result{
 		Catalog: c,

internal/mysql/parse.go

@@ -31,7 +31,7 @@ type Column struct {
 	Table string
 }
 
-func parsePath(sqlPath string, generator PackageGenerator) (*Result, error) {
+func parsePath(sqlPath []string, generator PackageGenerator) (*Result, error) {
 	files, err := dinosql.ReadSQLFiles(sqlPath)
 	if err != nil {
 		return nil, err
@@ -464,7 +464,7 @@ func (pGen PackageGenerator) parseSelectAliasExpr(exprs sqlparser.SelectExprs, t
 }
 
 // GeneratePkg is the main entry to mysql generator package
-func GeneratePkg(pkgName, schemaPath, querysPath string, settings config.CombinedSettings) (*Result, error) {
+func GeneratePkg(pkgName string, schemaPath, querysPath []string, settings config.CombinedSettings) (*Result, error) {
 	s := NewSchema()
 	generator := PackageGenerator{
 		Schema:           s,

internal/sqltest/postgres.go

@@ -29,7 +29,7 @@ func id() string {
 	return string(b)
 }
 
-func PostgreSQL(t *testing.T, migrations string) (*sql.DB, func()) {
+func PostgreSQL(t *testing.T, migrations []string) (*sql.DB, func()) {
 	t.Helper()
 
 	pgUser := os.Getenv("PG_USER")