feat(vet): Run rules against a managed database (#2751)

* feat(vet): Support managed databases
* feat(vet): Run rules against a managed database
* Add managed to JSON schema
* Use different hostname for region discovery
* docs: second pass at managed-databases.md

---------

Co-authored-by: Andrew Benton <andrew@sqlc.dev>

Author: kyleconroy

.github/workflows/ci.yml

@@ -32,16 +32,6 @@ jobs:
     runs-on: ubuntu-latest
 
     services:
-      postgres:
-        image: "postgres:15"
-        env:
-          POSTGRES_DB: postgres
-          POSTGRES_PASSWORD: postgres
-          POSTGRES_USER: postgres
-        ports:
-        - 5432:5432
-        # needed because the postgres container does not provide a healthcheck
-        options: --health-cmd pg_isready --health-interval 10s --health-timeout 5s --health-retries 5
       mysql:
         image: "mysql/mysql-server:8.0"
         env:
@@ -69,17 +59,13 @@ jobs:
     - name: test ./...
       run: gotestsum --junitfile junit.xml -- --tags=examples ./...
       env:
-        PG_USER: postgres
-        PG_HOST: localhost
-        PG_DATABASE: postgres
-        PG_PASSWORD: postgres
-        PG_PORT: ${{ job.services.postgres.ports['5432'] }}
         MYSQL_DATABASE: mysql
         MYSQL_HOST: localhost
         MYSQL_PORT: ${{ job.services.mysql.ports['3306'] }}
         MYSQL_ROOT_PASSWORD: mysecretpassword
         CI_SQLC_PROJECT_ID: ${{ secrets.CI_SQLC_PROJECT_ID }}
         CI_SQLC_AUTH_TOKEN: ${{ secrets.CI_SQLC_AUTH_TOKEN }}
+        SQLC_AUTH_TOKEN: ${{ secrets.CI_SQLC_AUTH_TOKEN }}
 
     - name: build internal/endtoend
       run: go build ./...

docs/howto/managed-databases.md

@@ -0,0 +1,68 @@
+# Managed databases
+
+*Added in v1.22.0*
+
+`sqlc` can create and maintain hosted databases for your project. These
+databases are immediately useful for linting queries with [`sqlc vet`](vet.md)
+if your lint rules require a connection to a running database. PostgreSQL
+support is available today, with MySQL on the way.
+
+This feature is under active development, and we're interested in supporting
+other use-cases. Beyond linting queries, you can use sqlc managed databases
+in your tests to quickly stand up a database per test suite or even per test,
+providing a real, isolated database for a test run. No cleanup required.
+
+Interested in trying out managed databases? Sign up [here](https://docs.google.com/forms/d/e/1FAIpQLSdxoMzJ7rKkBpuez-KyBcPNyckYV-5iMR--FRB7WnhvAmEvKg/viewform) or send us an email
+at <mailto:hello@sqlc.dev>.
+
+## Configuring managed databases
+
+To configure `sqlc` to use a managed database, remove the `uri` key from your
+`database` configuration and replace it with the `managed` key set to `true`.
+Set the `project` key in your `cloud` configuration to the value of your
+project ID, obtained via the sqlc.dev Dashboard.
+
+```yaml
+version: '2'
+cloud:
+  project: '<PROJECT_ID>'
+sql:
+- schema: schema.sql
+  queries: query.sql
+  engine: postgresql
+  database:
+    managed: true
+```
+
+## Authentication
+
+`sqlc` expects to find a valid auth token in the value of the `SQLC_AUTH_TOKEN`
+environment variable. You can create an auth token via the sqlc.dev Dashboard.
+
+```shell
+export SQLC_AUTH_TOKEN=sqlc_xxxxxxxx
+```
+
+## Linting queries
+
+With managed databases configured, `sqlc vet` will create a database with your
+package's schema and use that database when running lint rules that require a
+database connection, e.g. any [rule relying on `EXPLAIN ...` output](vet.md#rules-using-explain-output).
+
+If you don't yet have any vet rules, the [built-in sqlc/db-prepare rule](vet.md#sqlc-db-prepare)
+is a good place to start. It prepares each of your queries against the database
+to ensure the query is valid. Here's a minimal working configuration:
+
+```yaml
+version: '2'
+cloud:
+  project: '<PROJECT_ID>'
+sql:
+- schema: schema.sql
+  queries: query.sql
+  engine: postgresql
+  database:
+    managed: true
+  rules:
+  - sqlc/db-prepare
+```

examples/authors/sqlc.yaml

@@ -1,10 +1,12 @@
 version: '2'
+cloud:
+  project: "01HAQMMECEYQYKFJN8MP16QC41"
 sql:
 - schema: postgresql/schema.sql
   queries: postgresql/query.sql
   engine: postgresql
   database:
-    uri: postgresql://${PG_USER}:${PG_PASSWORD}@${PG_HOST}:${PG_PORT}/authors
+    managed: true
   rules:
   - sqlc/db-prepare
   - postgresql-query-too-costly

examples/batch/sqlc.json

@@ -1,5 +1,8 @@
 {
   "version": "1",
+  "cloud": {
+    "project": "01HAQMMECEYQYKFJN8MP16QC41"
+  },
   "packages": [
     {
       "path": "postgresql",
@@ -8,7 +11,7 @@
       "queries": "postgresql/query.sql",
       "engine": "postgresql",
       "database": {
-        "uri": "postgresql://${PG_USER}:${PG_PASSWORD}@${PG_HOST}:${PG_PORT}/batch"
+        "managed": true
       },
       "rules": [
         "sqlc/db-prepare"

examples/booktest/sqlc.json

@@ -1,5 +1,8 @@
 {
   "version": "1",
+  "cloud": {
+    "project": "01HAQMMECEYQYKFJN8MP16QC41"
+  },
   "packages": [
     {
       "name": "booktest",
@@ -8,7 +11,7 @@
       "queries": "postgresql/query.sql",
       "engine": "postgresql",
       "database": {
-        "uri": "postgresql://${PG_USER}:${PG_PASSWORD}@${PG_HOST}:${PG_PORT}/booktest"
+        "managed": true
       },
       "rules": [
         "sqlc/db-prepare"

examples/jets/sqlc.json

@@ -1,5 +1,8 @@
 {
   "version": "1",
+  "cloud": {
+    "project": "01HAQMMECEYQYKFJN8MP16QC41"
+  },
   "packages": [
     {
       "path": "postgresql",
@@ -8,7 +11,7 @@
       "queries": "postgresql/query-building.sql",
       "engine": "postgresql",
       "database": {
-        "uri": "postgresql://${PG_USER}:${PG_PASSWORD}@${PG_HOST}:${PG_PORT}/jets"
+        "managed": true
       },
       "rules": [
         "sqlc/db-prepare"

examples/ondeck/sqlc.json

@@ -1,5 +1,8 @@
 {
   "version": "1",
+  "cloud": {
+    "project": "01HAQMMECEYQYKFJN8MP16QC41"
+  },
   "packages": [
     {
       "path": "postgresql",
@@ -8,7 +11,7 @@
       "queries": "postgresql/query",
       "engine": "postgresql",
       "database": {
-        "uri": "postgresql://${PG_USER}:${PG_PASSWORD}@${PG_HOST}:${PG_PORT}/ondeck"
+        "managed": true
       },
       "rules": [
         "sqlc/db-prepare"

internal/cmd/cmd.go

@@ -37,6 +37,7 @@ func Do(args []string, stdin io.Reader, stdout io.Writer, stderr io.Writer) int
 	rootCmd.PersistentFlags().StringP("file", "f", "", "specify an alternate config file (default: sqlc.yaml)")
 	rootCmd.PersistentFlags().BoolP("experimental", "x", false, "DEPRECATED: enable experimental features (default: false)")
 	rootCmd.PersistentFlags().Bool("no-remote", false, "disable remote execution (default: false)")
+	rootCmd.PersistentFlags().Bool("remote", false, "enable remote execution (default: false)")
 	rootCmd.PersistentFlags().Bool("no-database", false, "disable database connections (default: false)")
 
 	rootCmd.AddCommand(checkCmd)
@@ -136,17 +137,20 @@ var initCmd = &cobra.Command{
 type Env struct {
 	DryRun     bool
 	Debug      opts.Debug
+	Remote     bool
 	NoRemote   bool
 	NoDatabase bool
 }
 
 func ParseEnv(c *cobra.Command) Env {
 	dr := c.Flag("dry-run")
+	r := c.Flag("remote")
 	nr := c.Flag("no-remote")
 	nodb := c.Flag("no-database")
 	return Env{
 		DryRun:     dr != nil && dr.Changed,
 		Debug:      opts.DebugFromEnv(),
+		Remote:     r != nil && nr.Value.String() == "true",
 		NoRemote:   nr != nil && nr.Value.String() == "true",
 		NoDatabase: nodb != nil && nodb.Value.String() == "true",
 	}

internal/cmd/generate.go

@@ -145,7 +145,7 @@ func Generate(ctx context.Context, e Env, dir, filename string, stderr io.Writer
 		return nil, err
 	}
 
-	if conf.Cloud.Project != "" && !e.NoRemote {
+	if conf.Cloud.Project != "" && e.Remote && !e.NoRemote {
 		return remoteGenerate(ctx, configPath, conf, dir, stderr)
 	}
 

internal/cmd/vet.go

@@ -26,7 +26,10 @@ import (
 	"github.com/sqlc-dev/sqlc/internal/debug"
 	"github.com/sqlc-dev/sqlc/internal/opts"
 	"github.com/sqlc-dev/sqlc/internal/plugin"
+	"github.com/sqlc-dev/sqlc/internal/quickdb"
+	pb "github.com/sqlc-dev/sqlc/internal/quickdb/v1"
 	"github.com/sqlc-dev/sqlc/internal/shfmt"
+	"github.com/sqlc-dev/sqlc/internal/sql/sqlpath"
 	"github.com/sqlc-dev/sqlc/internal/vet"
 )
 
@@ -376,6 +379,64 @@ type checker struct {
 	Envmap     map[string]string
 	Stderr     io.Writer
 	NoDatabase bool
+	Client     pb.QuickClient
+}
+
+func (c *checker) fetchDatabaseUri(ctx context.Context, s config.SQL) (string, func() error, error) {
+	cleanup := func() error {
+		return nil
+	}
+
+	if s.Database == nil {
+		panic("fetch database URI called with nil database")
+	}
+	if !s.Database.Managed {
+		uri, err := c.DSN(s.Database.URI)
+		return uri, cleanup, err
+	}
+	if s.Engine != config.EnginePostgreSQL {
+		return "", cleanup, fmt.Errorf("managed: only PostgreSQL currently")
+	}
+
+	if c.Client == nil {
+		// FIXME: Eventual race condition
+		client, err := quickdb.NewClientFromConfig(c.Conf.Cloud)
+		if err != nil {
+			return "", cleanup, fmt.Errorf("managed: client: %w", err)
+		}
+		c.Client = client
+	}
+
+	var migrations []string
+	files, err := sqlpath.Glob(s.Schema)
+	if err != nil {
+		return "", cleanup, err
+	}
+	for _, query := range files {
+		contents, err := os.ReadFile(query)
+		if err != nil {
+			return "", cleanup, fmt.Errorf("read file: %w", err)
+		}
+		migrations = append(migrations, string(contents))
+	}
+
+	resp, err := c.Client.CreateEphemeralDatabase(ctx, &pb.CreateEphemeralDatabaseRequest{
+		Engine:     "postgresql",
+		Region:     quickdb.GetClosestRegion(),
+		Migrations: migrations,
+	})
+	if err != nil {
+		return "", cleanup, fmt.Errorf("managed: create database: %w", err)
+	}
+
+	cleanup = func() error {
+		_, err := c.Client.DropEphemeralDatabase(ctx, &pb.DropEphemeralDatabaseRequest{
+			DatabaseId: resp.DatabaseId,
+		})
+		return err
+	}
+
+	return resp.Uri, cleanup, nil
 }
 
 func (c *checker) DSN(dsn string) (string, error) {
@@ -422,10 +483,16 @@ func (c *checker) checkSQL(ctx context.Context, s config.SQL) error {
 		if c.NoDatabase {
 			return fmt.Errorf("database: connections disabled via command line flag")
 		}
-		dburl, err := c.DSN(s.Database.URI)
+		dburl, cleanup, err := c.fetchDatabaseUri(ctx, s)
 		if err != nil {
 			return err
 		}
+		defer func() {
+			if err := cleanup(); err != nil {
+				fmt.Fprintf(c.Stderr, "error cleaning up: %s\n", err)
+			}
+		}()
+
 		switch s.Engine {
 		case config.EnginePostgreSQL:
 			conn, err := pgx.Connect(ctx, dburl)

internal/config/config.go

@@ -69,7 +69,8 @@ type Project struct {
 }
 
 type Database struct {
-	URI string `json:"uri" yaml:"uri"`
+	URI     string `json:"uri" yaml:"uri"`
+	Managed bool   `json:"managed" yaml:"managed"`
 }
 
 type Cloud struct {

internal/config/v_one.json

@@ -77,6 +77,9 @@
                         "properties": {
                             "uri": {
                                 "type": "string"
+                            },
+                            "managed": {
+                                "type": "boolean"
                             }
                         }
                     },

internal/config/v_two.json

@@ -77,6 +77,9 @@
                         "properties": {
                             "uri": {
                                 "type": "string"
+                            },
+                            "managed": {
+                                "type": "boolean"
                             }
                         }
                     },

internal/config/validate.go

@@ -12,8 +12,8 @@ func Validate(c *Config) error {
 			return fmt.Errorf("invalid config: emit_methods_with_db_argument and emit_prepared_queries settings are mutually exclusive")
 		}
 		if sql.Database != nil {
-			if sql.Database.URI == "" {
-				return fmt.Errorf("invalid config: database must have a non-empty URI")
+			if sql.Database.URI == "" && !sql.Database.Managed {
+				return fmt.Errorf("invalid config: database must be managed or have a non-empty URI")
 			}
 		}
 	}

internal/endtoend/vet_test.go

@@ -51,11 +51,6 @@ func TestExamplesVet(t *testing.T) {
 			path := filepath.Join(examples, tc)
 
 			if tc != "kotlin" && tc != "python" {
-				if s, found := findSchema(t, filepath.Join(path, "postgresql")); found {
-					db, cleanup := sqltest.CreatePostgreSQLDatabase(t, tc, false, []string{s})
-					defer db.Close()
-					defer cleanup()
-				}
 				if s, found := findSchema(t, filepath.Join(path, "mysql")); found {
 					db, cleanup := sqltest.CreateMySQLDatabase(t, tc, []string{s})
 					defer db.Close()

internal/quickdb/region.go

@@ -10,9 +10,9 @@ var once sync.Once
 
 func GetClosestRegion() string {
 	once.Do(func() {
-		resp, err := http.Get("https://debug.fly.dev")
+		resp, err := http.Get("https://find-closest-db-region.sqlc.dev")
 		if err == nil {
-			region = resp.Header.Get("Fly-Region")
+			region = resp.Header.Get("Region")
 		}
 	})
 	return region