feat(codegen): Allow plugins to access environment variables (#2669)

Allow plugins, both process-based and WASM-based, to read environment variables from the host, configured via a per-variable allowlist. Plugins can not declare the environment variables they have access to outside of the sqlc.yaml configuration file.

```yaml
version: '2'
plugins:
- name: py
  env:
  - PATH
  wasm:
    url: https://github.com/sqlc-dev/sqlc-gen-python/releases/download/v0.16.0-alpha/sqlc-gen-python.wasm
    sha256: 428476c7408fd4c032da4ec74e8a7344f4fa75e0f98a5a3302f238283b9b95f2
```

* Add SQLC_VERSION to docs

Author: kyleconroy

.github/workflows/ci.yml

@@ -55,9 +55,12 @@ jobs:
       with:
         go-version: '1.21.0'
 
-    - name: gotestsum
+    - name: install gotestsum
       run: go install gotest.tools/gotestsum@latest
 
+    - name: install sqlc-gen-test
+      run: go install github.com/sqlc-dev/sqlc-gen-test@v0.1.0
+
     - name: install ./...
       run: go install ./...
 

docs/guides/plugins.md

@@ -15,32 +15,20 @@ top-level `plugins` map. The `options` are serialized to a string and passed on
 to the plugin itself.
 
 
-```json
-{
-  "version": "2",
-  "plugins": [
-    {
-      "name": "greeter",
-      "wasm": {
-        "url": "https://github.com/sqlc-dev/sqlc-gen-greeter/releases/download/v0.1.0/sqlc-gen-greeter.wasm",
-        "sha256": "afc486dac2068d741d7a4110146559d12a013fd0286f42a2fc7dcd802424ad07"
-      }
-    }
-  ],
-  "sql": [
-    {
-      "schema": "schema.sql",
-      "queries": "query.sql",
-      "engine": "postgresql",
-      "codegen": [
-        {
-          "out": "gen",
-          "plugin": "greeter"
-        }
-      ]
-    }
-  ]
-}
+```yaml
+version: '2'
+plugins:
+- name: greeter
+  wasm:
+    url: https://github.com/sqlc-dev/sqlc-gen-greeter/releases/download/v0.1.0/sqlc-gen-greeter.wasm
+    sha256: afc486dac2068d741d7a4110146559d12a013fd0286f42a2fc7dcd802424ad07
+sql:
+- schema: schema.sql
+  queries: query.sql
+  engine: postgresql
+  codegen:
+  - out: gen
+    plugin: greeter
 ```
 
 For a complete working example see the following files:
@@ -59,39 +47,54 @@ the new files. The `plugin` key must reference a plugin defined in the
 top-level `plugins` map. The `options` are serialized to a string and passed on
 to the plugin itself.
 
-```json
-{
-  "version": "2",
-  "plugins": [
-    {
-      "name": "jsonb",
-      "process": {
-        "cmd": "sqlc-gen-json"
-      }
-    }
-  ],
-   "sql": [
-    {
-      "schema": "schema.sql",
-      "queries": "query.sql",
-      "engine": "postgresql",
-      "codegen": [
-        {
-          "out": "gen",
-          "plugin": "jsonb",
-          "options": {
-            "indent": "  ",
-            "filename": "codegen.json"
-          }
-        }
-      ]
-    }
-  ]
-}
+```yaml
+version: '2'
+plugins:
+- name: jsonb
+  process:
+    cmd: sqlc-gen-json
+sql:
+- schema: schema.sql
+  queries: query.sql
+  engine: postgresql
+  codegen:
+  - out: gen
+    plugin: jsonb
+    options:
+      indent: "  "
+      filename: codegen.json
 ```
 
 For a complete working example see the following files:
 - [sqlc-gen-json](https://github.com/sqlc-dev/sqlc/tree/main/cmd/sqlc-gen-json)
   - A process-based plugin that serializes the CodeGenRequest to JSON
 - [process_plugin_sqlc_gen_json](https://github.com/sqlc-dev/sqlc/tree/main/internal/endtoend/testdata/process_plugin_sqlc_gen_json)
   - An example project showing how to use a process-based plugin
+
+## Environment variables
+
+By default, plugins do not inherit access to environment variables. Instead,
+access can be configured on a per-variable basis. For example, if your plugin
+needs the `PATH` environment variable, add the `PATH` to the `env` option in the
+`plugins` collection.
+
+```yaml
+version: '2'
+sql:
+- schema: schema.sql
+  queries: query.sql
+  engine: postgresql
+  codegen:
+  - out: gen
+    plugin: test
+plugins:
+- name: test
+  env:
+  - PATH
+  wasm:
+    url: https://github.com/sqlc-dev/sqlc-gen-test/releases/download/v0.1.0/sqlc-gen-test.wasm
+    sha256: 138220eae508d4b65a5a8cea555edd155eb2290daf576b7a8b96949acfeb3790
+```
+
+The `SQLC_VERSION` environment variable will always be included in the
+environment, set to the version of the `sqlc` executable.

docs/reference/config.md

@@ -316,6 +316,8 @@ Each mapping in the `plugins` collection has the following keys:
 
 - `name`:
   - The name of this plugin. Required
+- `env`
+  - A list of environment variables to pass to the plugin. By default, no environment variables are passed.
 - `process`: A mapping with a single `cmd` key
   - `cmd`:
     - The executable to call when using this plugin
@@ -333,6 +335,8 @@ plugins:
     url: "https://github.com/sqlc-dev/sqlc-gen-python/releases/download/v0.16.0-alpha/sqlc-gen-python.wasm"
     sha256: "428476c7408fd4c032da4ec74e8a7344f4fa75e0f98a5a3302f238283b9b95f2"
 - name: "js"
+  env:
+  - PATH
   process: 
     cmd: "sqlc-gen-json"
 ```

internal/cmd/generate.go

@@ -386,11 +386,13 @@ func codegen(ctx context.Context, combo config.CombinedSettings, sql outPair, re
 		case plug.Process != nil:
 			handler = &process.Runner{
 				Cmd: plug.Process.Cmd,
+				Env: plug.Env,
 			}
 		case plug.WASM != nil:
 			handler = &wasm.Runner{
 				URL:    plug.WASM.URL,
 				SHA256: plug.WASM.SHA256,
+				Env:    plug.Env,
 			}
 		default:
 			return "", nil, fmt.Errorf("unsupported plugin type")

internal/config/config.go

@@ -79,7 +79,8 @@ type Cloud struct {
 }
 
 type Plugin struct {
-	Name    string `json:"name" yaml:"name"`
+	Name    string   `json:"name" yaml:"name"`
+	Env     []string `json:"env" yaml:"env"`
 	Process *struct {
 		Cmd string `json:"cmd" yaml:"cmd"`
 	} `json:"process" yaml:"process"`

internal/endtoend/endtoend_test.go

@@ -75,6 +75,10 @@ func BenchmarkExamples(b *testing.B) {
 }
 
 func TestReplay(t *testing.T) {
+	// Ensure that this environment variable is always set to true when running
+	// end-to-end tests
+	os.Setenv("SQLC_DUMMY_VALUE", "true")
+
 	t.Parallel()
 	ctx := context.Background()
 	var dirs []string

internal/endtoend/testdata/process_plugin_sqlc_gen_test/exec.json

@@ -0,0 +1,3 @@
+{
+  "process": "sqlc-gen-test"
+}

internal/endtoend/testdata/process_plugin_sqlc_gen_test/gen/env.json

@@ -0,0 +1,6 @@
+{
+  "env": [
+    "SQLC_VERSION=v1.20.0",
+    "SQLC_DUMMY_VALUE=true"
+  ]
+}

internal/endtoend/testdata/process_plugin_sqlc_gen_test/query.sql

@@ -0,0 +1,19 @@
+-- name: GetAuthor :one
+SELECT * FROM authors
+WHERE id = $1 LIMIT 1;
+
+-- name: ListAuthors :many
+SELECT * FROM authors
+ORDER BY name;
+
+-- name: CreateAuthor :one
+INSERT INTO authors (
+          name, bio
+) VALUES (
+  $1, $2
+)
+RETURNING *;
+
+-- name: DeleteAuthor :exec
+DELETE FROM authors
+WHERE id = $1;

internal/endtoend/testdata/process_plugin_sqlc_gen_test/schema.sql

@@ -0,0 +1,5 @@
+CREATE TABLE authors (
+          id   BIGSERIAL PRIMARY KEY,
+          name text      NOT NULL,
+          bio  text
+);

internal/endtoend/testdata/process_plugin_sqlc_gen_test/sqlc.json

@@ -0,0 +1,25 @@
+{
+  "version": "2",
+  "sql": [
+    {
+      "schema": "schema.sql",
+      "queries": "query.sql",
+      "engine": "postgresql",
+      "codegen": [
+        {
+          "out": "gen",
+          "plugin": "test"
+        }
+      ]
+    }
+  ],
+  "plugins": [
+    {
+      "name": "test",
+      "env": ["SQLC_DUMMY_VALUE"],
+      "process": {
+        "cmd": "sqlc-gen-test"
+      }
+    }
+  ]
+}

internal/endtoend/testdata/wasm_plugin_sqlc_gen_test/gen/env.json

@@ -0,0 +1,6 @@
+{
+  "env": [
+    "SQLC_VERSION=v1.20.0",
+    "SQLC_DUMMY_VALUE=true"
+  ]
+}

internal/endtoend/testdata/wasm_plugin_sqlc_gen_test/query.sql

@@ -0,0 +1,19 @@
+-- name: GetAuthor :one
+SELECT * FROM authors
+WHERE id = $1 LIMIT 1;
+
+-- name: ListAuthors :many
+SELECT * FROM authors
+ORDER BY name;
+
+-- name: CreateAuthor :one
+INSERT INTO authors (
+          name, bio
+) VALUES (
+  $1, $2
+)
+RETURNING *;
+
+-- name: DeleteAuthor :exec
+DELETE FROM authors
+WHERE id = $1;

internal/endtoend/testdata/wasm_plugin_sqlc_gen_test/schema.sql

@@ -0,0 +1,5 @@
+CREATE TABLE authors (
+          id   BIGSERIAL PRIMARY KEY,
+          name text      NOT NULL,
+          bio  text
+);

internal/endtoend/testdata/wasm_plugin_sqlc_gen_test/sqlc.json

@@ -0,0 +1,26 @@
+{
+  "version": "2",
+  "sql": [
+    {
+      "schema": "schema.sql",
+      "queries": "query.sql",
+      "engine": "postgresql",
+      "codegen": [
+        {
+          "out": "gen",
+          "plugin": "test"
+        }
+      ]
+    }
+  ],
+  "plugins": [
+    {
+      "name": "test",
+      "env": ["SQLC_DUMMY_VALUE"],
+      "wasm": {
+        "url": "https://github.com/sqlc-dev/sqlc-gen-test/releases/download/v0.1.0/sqlc-gen-test.wasm",
+        "sha256": "138220eae508d4b65a5a8cea555edd155eb2290daf576b7a8b96949acfeb3790"
+      }
+    }
+  ]
+}

internal/ext/process/gen.go

@@ -5,6 +5,7 @@ import (
 	"context"
 	"errors"
 	"fmt"
+	"os"
 	"os/exec"
 
 	"google.golang.org/protobuf/proto"
@@ -14,6 +15,7 @@ import (
 
 type Runner struct {
 	Cmd string
+	Env []string
 }
 
 // TODO: Update the gen func signature to take a ctx
@@ -34,6 +36,9 @@ func (r Runner) Generate(ctx context.Context, req *plugin.CodeGenRequest) (*plug
 	cmd.Env = []string{
 		fmt.Sprintf("SQLC_VERSION=%s", req.SqlcVersion),
 	}
+	for _, key := range r.Env {
+		cmd.Env = append(cmd.Env, fmt.Sprintf("%s=%s", key, os.Getenv(key)))
+	}
 
 	out, err := cmd.Output()
 	if err != nil {

internal/ext/wasm/wasm.go

@@ -47,6 +47,7 @@ func cacheDir() (string, error) {
 type Runner struct {
 	URL    string
 	SHA256 string
+	Env    []string
 }
 
 var flight singleflight.Group
@@ -255,6 +256,14 @@ func (r *Runner) Generate(ctx context.Context, req *plugin.CodeGenRequest) (*plu
 	wasiConfig.SetStdoutFile(stdoutPath)
 	wasiConfig.SetStderrFile(stderrPath)
 
+	keys := []string{"SQLC_VERSION"}
+	vals := []string{req.SqlcVersion}
+	for _, key := range r.Env {
+		keys = append(keys, key)
+		vals = append(vals, os.Getenv(key))
+	}
+	wasiConfig.SetEnv(keys, vals)
+
 	store := wasmtime.NewStore(engine)
 	store.SetWasi(wasiConfig)
 

scripts/regenerate/main.go

@@ -56,6 +56,7 @@ func regenerate(dir string) error {
 			}
 
 			cmd := exec.Command("sqlc-dev", "generate")
+			cmd.Env = append(cmd.Env, "SQLC_DUMMY_VALUE=true")
 			cmd.Dir = cwd
 			out, failed := cmd.CombinedOutput()
 			if failed != nil && !expectFailure {