Repository: sqlc-dev/sqlc (go)

Issue:
Have process plugins support json as input and output
### What do you want to change?

Example config

```
{
  "version": "2",
  "sql": [
    {
      "schema": "schema.sql",
      "queries": "query.sql",
      "engine": "postgresql",
      "codegen": [
        {
          "out": "gen",
          "plugin": "jsonb"
        }
      ]
    }
  ],
  "plugins": [
    {
      "name": "jsonb",
      "process": {
        "cmd": "test_json_process_plugin",
        "format": "json"
      }
    }
  ]
}
```

### What database engines need to be changed?

_No response_

### What programming language backends need to be changed?

_No response_

Pull request description:
Add support for json format from process plugins
Fixes #3558 

Patch:
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 8aec33c8e..34d625bf7 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -38,6 +38,9 @@ jobs:
     - name: install sqlc-gen-test
       run: go install github.com/sqlc-dev/sqlc-gen-test@v0.1.0
 
+    - name: install test-json-process-plugin
+      run: go install ./scripts/test-json-process-plugin/
+
     - name: install ./...
       run: go install ./...
       env:
@@ -75,9 +78,4 @@ jobs:
     timeout-minutes: 5
 
     steps:
-    - uses: actions/checkout@v4
-    - uses: actions/setup-go@v5
-      with:
-        go-version: '1.23.5'
-    - run: go install golang.org/x/vuln/cmd/govulncheck@latest
-    - run: govulncheck ./...
+    - uses: golang/govulncheck-action@v1
diff --git a/Dockerfile b/Dockerfile
index 2e8e74473..9627f2b10 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -1,5 +1,5 @@
 # STEP 1: Build sqlc
-FROM golang:1.23.5 AS builder
+FROM golang:1.23.6 AS builder
 
 COPY . /workspace
 WORKDIR /workspace
diff --git a/Makefile b/Makefile
index e9c2fbb89..b8745e57d 100644
--- a/Makefile
+++ b/Makefile
@@ -32,6 +32,9 @@ sqlc-pg-gen:
 sqlc-gen-json:
 	go build -o ~/bin/sqlc-gen-json ./cmd/sqlc-gen-json
 
+test-json-process-plugin:
+	go build -o ~/bin/test-json-process-plugin ./scripts/test-json-process-plugin/
+
 start:
 	docker compose up -d
 
diff --git a/docs/guides/plugins.md b/docs/guides/plugins.md
index 0a97a2b77..b229f9173 100644
--- a/docs/guides/plugins.md
+++ b/docs/guides/plugins.md
@@ -72,6 +72,8 @@ For a complete working example see the following files:
   - A process-based plugin that serializes the CodeGenRequest to JSON
 - [process_plugin_sqlc_gen_json](https://github.com/sqlc-dev/sqlc/tree/main/internal/endtoend/testdata/process_plugin_sqlc_gen_json)
   - An example project showing how to use a process-based plugin
+- [process_plugin_sqlc_gen_json](https://github.com/sqlc-dev/sqlc/tree/main/internal/endtoend/testdata/process_plugin_format_json/)
+  - An example project showing how to use a process-based plugin using json
 
 ## Environment variables
 
@@ -99,4 +101,4 @@ plugins:
 ```
 
 A variable named `SQLC_VERSION` is always included in the plugin's
-environment, set to the version of the `sqlc` executable invoking it.
\ No newline at end of file
+environment, set to the version of the `sqlc` executable invoking it.
diff --git a/docs/reference/config.md b/docs/reference/config.md
index 9d334834f..e6e690b6a 100644
--- a/docs/reference/config.md
+++ b/docs/reference/config.md
@@ -273,6 +273,8 @@ Each mapping in the `plugins` collection has the following keys:
 - `process`: A mapping with a single `cmd` key
   - `cmd`:
     - The executable to call when using this plugin
+  - `format`:
+    - The format expected. Supports `json` and `protobuf` formats. Defaults to `protobuf`.
 - `wasm`: A mapping with a two keys `url` and `sha256`
   - `url`:
     - The URL to fetch the WASM file. Supports the `https://` or `file://` schemes.
diff --git a/docs/reference/datatypes.md b/docs/reference/datatypes.md
index a440be657..bb3d8168b 100644
--- a/docs/reference/datatypes.md
+++ b/docs/reference/datatypes.md
@@ -235,7 +235,7 @@ create table shapes(
 );
 
 -- name: GetCentroids :many
-SELECT id, name, ST_Centriod(geom)::geometry FROM shapes;
+SELECT id, name, ST_Centroid(geom)::geometry FROM shapes;
 ```
 
 ```json
diff --git a/docs/reference/language-support.rst b/docs/reference/language-support.rst
index 13256c951..57391ddb3 100644
--- a/docs/reference/language-support.rst
+++ b/docs/reference/language-support.rst
@@ -23,6 +23,7 @@ F#        `kaashyapan/sqlc-gen-fsharp`_      Not implemented  Beta             B
 C#        `DaredevilOSS/sqlc-gen-csharp`_    Beta             Beta             Beta
 Ruby      `DaredevilOSS/sqlc-gen-ruby`_      Beta             Beta             Beta
 PHP       `lcarilla/sqlc-plugin-php-dbal`_   Beta             Not implemented  Not implemented
+Zig       `tinyzimmer/sqlc-gen-zig`_         Not implemented  Beta             Not implemented
 [Any]     `fdietze/sqlc-gen-from-template`_  Stable           Stable           Stable
 ========  =================================  ===============  ===============  ===============
 
@@ -35,6 +36,7 @@ PHP       `lcarilla/sqlc-plugin-php-dbal`_   Beta             Not implemented  N
 .. _DaredevilOSS/sqlc-gen-ruby: https://github.com/DaredevilOSS/sqlc-gen-ruby
 .. _fdietze/sqlc-gen-from-template: https://github.com/fdietze/sqlc-gen-from-template
 .. _lcarilla/sqlc-plugin-php-dbal: https://github.com/lcarilla/sqlc-plugin-php-dbal
+.. _tinyzimmer/sqlc-gen-zig: https://github.com/tinyzimmer/sqlc-gen-zig
 
 Future language support
 ************************
diff --git a/docs/requirements.txt b/docs/requirements.txt
index 3a279bc71..135b70826 100644
--- a/docs/requirements.txt
+++ b/docs/requirements.txt
@@ -1,9 +1,9 @@
-Babel==2.16.0
+Babel==2.17.0
 Jinja2==3.1.5
 MarkupSafe==3.0.2
 Pygments==2.19.1
 Sphinx==7.4.7
-certifi==2024.12.14
+certifi==2025.1.31
 chardet==5.2.0
 commonmark==0.9.1
 docutils==0.20.1
@@ -12,7 +12,7 @@ imagesize==1.4.1
 myst-parser==4.0.0 
 packaging==24.2
 pyparsing==3.2.1
-pytz==2024.2
+pytz==2025.1
 requests==2.32.3
 snowballstemmer==2.2.0
 sphinx-favicon==1.0.1
diff --git a/internal/cmd/generate.go b/internal/cmd/generate.go
index a7e64e1e4..00e8871c7 100644
--- a/internal/cmd/generate.go
+++ b/internal/cmd/generate.go
@@ -349,8 +349,9 @@ func codegen(ctx context.Context, combo config.CombinedSettings, sql OutputPair,
 		switch {
 		case plug.Process != nil:
 			handler = &process.Runner{
-				Cmd: plug.Process.Cmd,
-				Env: plug.Env,
+				Cmd:    plug.Process.Cmd,
+				Env:    plug.Env,
+				Format: plug.Process.Format,
 			}
 		case plug.WASM != nil:
 			handler = &wasm.Runner{
diff --git a/internal/config/config.go b/internal/config/config.go
index 5bfa506b0..0ff805fcc 100644
--- a/internal/config/config.go
+++ b/internal/config/config.go
@@ -89,7 +89,8 @@ type Plugin struct {
 	Name    string   `json:"name" yaml:"name"`
 	Env     []string `json:"env" yaml:"env"`
 	Process *struct {
-		Cmd string `json:"cmd" yaml:"cmd"`
+		Cmd    string `json:"cmd" yaml:"cmd"`
+		Format string `json:"format" yaml:"format"`
 	} `json:"process" yaml:"process"`
 	WASM *struct {
 		URL    string `json:"url" yaml:"url"`
diff --git a/internal/ext/process/gen.go b/internal/ext/process/gen.go
index b677f6805..b5720dbc3 100644
--- a/internal/ext/process/gen.go
+++ b/internal/ext/process/gen.go
@@ -11,6 +11,7 @@ import (
 	"google.golang.org/grpc"
 	"google.golang.org/grpc/codes"
 	"google.golang.org/grpc/status"
+	"google.golang.org/protobuf/encoding/protojson"
 	"google.golang.org/protobuf/proto"
 	"google.golang.org/protobuf/reflect/protoreflect"
 
@@ -18,8 +19,9 @@ import (
 )
 
 type Runner struct {
-	Cmd string
-	Env []string
+	Cmd    string
+	Format string
+	Env    []string
 }
 
 func (r *Runner) Invoke(ctx context.Context, method string, args any, reply any, opts ...grpc.CallOption) error {
@@ -28,9 +30,27 @@ func (r *Runner) Invoke(ctx context.Context, method string, args any, reply any,
 		return fmt.Errorf("args isn't a protoreflect.ProtoMessage")
 	}
 
-	stdin, err := proto.Marshal(req)
-	if err != nil {
-		return fmt.Errorf("failed to encode codegen request: %w", err)
+	var stdin []byte
+	var err error
+	switch r.Format {
+	case "json":
+		m := &protojson.MarshalOptions{
+			EmitUnpopulated: true,
+			Indent:          "",
+			UseProtoNames:   true,
+		}
+		stdin, err = m.Marshal(req)
+
+		if err != nil {
+			return fmt.Errorf("failed to encode codegen request: %w", err)
+		}
+	case "", "protobuf":
+		stdin, err = proto.Marshal(req)
+		if err != nil {
+			return fmt.Errorf("failed to encode codegen request: %w", err)
+		}
+	default:
+		return fmt.Errorf("unknown plugin format: %s", r.Format)
 	}
 
 	// Check if the output plugin exists
@@ -66,8 +86,15 @@ func (r *Runner) Invoke(ctx context.Context, method string, args any, reply any,
 		return fmt.Errorf("reply isn't a protoreflect.ProtoMessage")
 	}
 
-	if err := proto.Unmarshal(out, resp); err != nil {
-		return fmt.Errorf("process: failed to read codegen resp: %w", err)
+	switch r.Format {
+	case "json":
+		if err := protojson.Unmarshal(out, resp); err != nil {
+			return fmt.Errorf("process: failed to read codegen resp: %w", err)
+		}
+	default:
+		if err := proto.Unmarshal(out, resp); err != nil {
+			return fmt.Errorf("process: failed to read codegen resp: %w", err)
+		}
 	}
 
 	return nil


Test patch:
diff --git a/examples/authors/sqlite/db_test.go b/examples/authors/sqlite/db_test.go
index 84f193959..e10ce4606 100644
--- a/examples/authors/sqlite/db_test.go
+++ b/examples/authors/sqlite/db_test.go
@@ -12,6 +12,7 @@ import (
 
 func TestAuthors(t *testing.T) {
 	sdb, cleanup := sqltest.SQLite(t, []string{"schema.sql"})
+	defer sdb.Close()
 	defer cleanup()
 
 	ctx := context.Background()
diff --git a/examples/booktest/sqlite/db_test.go b/examples/booktest/sqlite/db_test.go
index b9490245b..42a8b396b 100644
--- a/examples/booktest/sqlite/db_test.go
+++ b/examples/booktest/sqlite/db_test.go
@@ -18,6 +18,7 @@ const (
 
 func TestBooks(t *testing.T) {
 	db, cleanup := sqltest.SQLite(t, []string{"schema.sql"})
+	defer db.Close()
 	defer cleanup()
 
 	ctx := context.Background()
diff --git a/examples/ondeck/sqlite/db_test.go b/examples/ondeck/sqlite/db_test.go
index 39264c26d..e4bb4e419 100644
--- a/examples/ondeck/sqlite/db_test.go
+++ b/examples/ondeck/sqlite/db_test.go
@@ -149,6 +149,7 @@ func TestPrepared(t *testing.T) {
 	t.Parallel()
 
 	sdb, cleanup := sqltest.SQLite(t, []string{"schema"})
+	defer sdb.Close()
 	defer cleanup()
 
 	q, err := Prepare(context.Background(), sdb)
@@ -163,6 +164,7 @@ func TestQueries(t *testing.T) {
 	t.Parallel()
 
 	sdb, cleanup := sqltest.SQLite(t, []string{"schema"})
+	defer sdb.Close()
 	defer cleanup()
 
 	runOnDeckQueries(t, New(sdb))
diff --git a/internal/endtoend/testdata/process_plugin_format_json/exec.json b/internal/endtoend/testdata/process_plugin_format_json/exec.json
new file mode 100644
index 000000000..2cfb9f266
--- /dev/null
+++ b/internal/endtoend/testdata/process_plugin_format_json/exec.json
@@ -0,0 +1,4 @@
+{
+  "process": "test-json-process-plugin",
+  "os": [ "darwin", "linux" ]
+}
diff --git a/internal/endtoend/testdata/process_plugin_format_json/gen/hello.txt b/internal/endtoend/testdata/process_plugin_format_json/gen/hello.txt
new file mode 100644
index 000000000..05d6799f1
--- /dev/null
+++ b/internal/endtoend/testdata/process_plugin_format_json/gen/hello.txt
@@ -0,0 +1,12 @@
+SELECT id, name, bio FROM authors
+WHERE id = $1 LIMIT 1
+SELECT id, name, bio FROM authors
+ORDER BY name
+INSERT INTO authors (
+          name, bio
+) VALUES (
+  $1, $2
+)
+RETURNING id, name, bio
+DELETE FROM authors
+WHERE id = $1
diff --git a/internal/endtoend/testdata/process_plugin_format_json/query.sql b/internal/endtoend/testdata/process_plugin_format_json/query.sql
new file mode 100644
index 000000000..75e38b2ca
--- /dev/null
+++ b/internal/endtoend/testdata/process_plugin_format_json/query.sql
@@ -0,0 +1,19 @@
+-- name: GetAuthor :one
+SELECT * FROM authors
+WHERE id = $1 LIMIT 1;
+
+-- name: ListAuthors :many
+SELECT * FROM authors
+ORDER BY name;
+
+-- name: CreateAuthor :one
+INSERT INTO authors (
+          name, bio
+) VALUES (
+  $1, $2
+)
+RETURNING *;
+
+-- name: DeleteAuthor :exec
+DELETE FROM authors
+WHERE id = $1;
diff --git a/internal/endtoend/testdata/process_plugin_format_json/schema.sql b/internal/endtoend/testdata/process_plugin_format_json/schema.sql
new file mode 100644
index 000000000..b4fad7849
--- /dev/null
+++ b/internal/endtoend/testdata/process_plugin_format_json/schema.sql
@@ -0,0 +1,5 @@
+CREATE TABLE authors (
+          id   BIGSERIAL PRIMARY KEY,
+          name text      NOT NULL,
+          bio  text
+);
diff --git a/internal/endtoend/testdata/process_plugin_format_json/sqlc.json b/internal/endtoend/testdata/process_plugin_format_json/sqlc.json
new file mode 100644
index 000000000..28055fa57
--- /dev/null
+++ b/internal/endtoend/testdata/process_plugin_format_json/sqlc.json
@@ -0,0 +1,25 @@
+{
+  "version": "2",
+  "sql": [
+    {
+      "schema": "schema.sql",
+      "queries": "query.sql",
+      "engine": "postgresql",
+      "codegen": [
+        {
+          "out": "gen",
+          "plugin": "jsonb"
+        }
+      ]
+    }
+  ],
+  "plugins": [
+    {
+      "name": "jsonb",
+      "process": {
+        "cmd": "test-json-process-plugin",
+        "format": "json"
+      }
+    }
+  ]
+}
diff --git a/internal/sqltest/sqlite.go b/internal/sqltest/sqlite.go
index 72a154d0e..0e5161967 100644
--- a/internal/sqltest/sqlite.go
+++ b/internal/sqltest/sqlite.go
@@ -16,6 +16,9 @@ func SQLite(t *testing.T, migrations []string) (*sql.DB, func()) {
 	if err != nil {
 		t.Fatal(err)
 	}
+	if err := source.Close(); err != nil {
+		t.Fatal(err)
+	}
 	return CreateSQLiteDatabase(t, source.Name(), migrations)
 }
 
diff --git a/scripts/test-json-process-plugin/main.go b/scripts/test-json-process-plugin/main.go
new file mode 100644
index 000000000..6bcc7a25d
--- /dev/null
+++ b/scripts/test-json-process-plugin/main.go
@@ -0,0 +1,39 @@
+package main
+
+import (
+	"bytes"
+	"encoding/json"
+	"fmt"
+	"os"
+)
+
+type Out struct {
+	Files []File `json:"files"`
+}
+
+type File struct {
+	Name     string `json:"name"`
+	Contents []byte `json:"contents"`
+}
+
+func main() {
+	in := make(map[string]interface{})
+	decoder := json.NewDecoder(os.Stdin)
+	err := decoder.Decode(&in)
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "error generating JSON: %s", err)
+		os.Exit(2)
+	}
+
+	buf := bytes.NewBuffer(nil)
+	queries := in["queries"].([]interface{})
+	for _, q := range queries {
+		text := q.(map[string]interface{})["text"].(string)
+		buf.WriteString(text)
+		buf.WriteString("\n")
+	}
+
+	e := json.NewEncoder(os.Stdout)
+	e.SetIndent("", "  ")
+	e.Encode(&Out{Files: []File{{Name: "hello.txt", Contents: buf.Bytes()}}})
+}