Repository: kyleconroy/sqlc (go)

Issue:
Add support for YAML
Now that version 2 of the configuration format has been merged, we should add support for reading from `sqlc.yaml`. The newer format is a bit of a chore to write in JSON.

Pull request:
config: Add support for YAML
Both version 1 and version 2 of the config formats can now be expressed
in YAML. sqlc will look for `sqlc.yaml` defined in the working directory.

Fixes #330 

Patch:
diff --git a/README.md b/README.md
index 20e718f33..f4a84eaa6 100644
--- a/README.md
+++ b/README.md
@@ -256,7 +256,7 @@ Available Commands:
   compile     Statically check SQL for syntax and type errors
   generate    Generate Go code from SQL
   help        Help about any command
-  init        Create an empty sqlc.json settings file
+  init        Create an empty sqlc.yaml settings file
   version     Print the sqlc version number
 
 Flags:
@@ -267,24 +267,19 @@ Use "sqlc [command] --help" for more information about a command.
 
 ## Settings
 
-The `sqlc` tool is configured via a `sqlc.json` file. This file must be
+The `sqlc` tool is configured via a `sqlc.yaml` file. This file must be
 in the directory where the `sqlc` command is run.
 
-```json
-{
-  "version": "1",
-  "packages": [
-    {
-      "name": "db",
-      "emit_json_tags": true,
-      "emit_prepared_queries": false,
-      "emit_interface": true,
-      "path": "internal/db",
-      "queries": "./sql/query/",
-      "schema": "./sql/schema/"
-    }
-  ]
-}
+```yaml
+version: "1"
+packages:
+  - name: "db",
+    emit_json_tags: true
+    emit_prepared_queries: false
+    emit_interface: true
+    path: "internal/db"
+    queries: "./sql/query/"
+    schema: "./sql/schema/"
 ```
 
 Each package document has the following keys:
@@ -315,17 +310,12 @@ If a different Go package for UUIDs is required, specify the package in the
 `overrides` array. In this case, I'm going to use the `github.com/gofrs/uuid`
 instead.
 
-```
-{
-  "version": "1",
-  "packages": [...],
-  "overrides": [
-      {
-          "go_type": "github.com/gofrs/uuid.UUID",
-          "db_type": "uuid"
-      }
-  ]
-}
+```yaml
+version: "1"
+packages: [...]
+overrides:
+  - go_type: "github.com/gofrs/uuid.UUID"
+    db_type: "uuid"
 ```
 
 Each override document has the following keys:
@@ -345,17 +335,12 @@ This may be configured by specifying the `column` property in the override defin
 should be of the form `table.column` buy you may be even more specify by specifying `schema.table.column`
 or `catalog.schema.table.column`.
 
-```
-{
-  "version": "1",
-  "packages": [...],
-  "overrides": [
-    {
-      "column": "authors.id",
-      "go_type": "github.com/segmentio/ksuid.KSUID"
-    }
-  ]
-}
+```yaml
+version: "1"
+packages: [...]
+overrides:
+  - column: "authors.id"
+    go_type: "github.com/segmentio/ksuid.KSUID"
 ```
 
 ### Package Level Overrides
@@ -363,19 +348,12 @@ or `catalog.schema.table.column`.
 Overrides can be configured globally, as demonstrated in the previous sections, or they can be configured on a per-package which
 scopes the override behavior to just a single package:
 
-```
-{
-  "version": "1",
-  "packages": [
-    {
-      ...
-      "overrides": [...]
-    }
-  ],
-}
+```yaml
+version: "1"
+packages: 
+  - overrides: [...]
 ```
 
-
 ### Renaming Struct Fields
 
 Struct field names are generated from column names using a simple algorithm:
@@ -392,14 +370,11 @@ If you're not happy with a field's generated name, use the `rename` dictionary
 to pick a new name. The keys are column names and the values are the struct
 field name to use.
 
-```json
-{
-  "version": "1",
-  "packages": [...],
-  "rename": {
-    "spotify_url": "SpotifyURL"
-  }
-}
+```yaml
+version: "1"
+packages: [...]
+rename:
+  spotify_url: "SpotifyURL"
 ```
 
 ## Downloads
diff --git a/go.mod b/go.mod
index 412794ea4..6b64b66b0 100644
--- a/go.mod
+++ b/go.mod
@@ -13,5 +13,6 @@ require (
 	golang.org/x/sys v0.0.0-20191220220014-0732a990476f // indirect
 	google.golang.org/genproto v0.0.0-20191223191004-3caeed10a8bf // indirect
 	google.golang.org/grpc v1.26.0 // indirect
+	gopkg.in/yaml.v3 v3.0.0-20200121175148-a6ecf24a6d71
 	vitess.io/vitess v0.0.0-20200119095853-bd8205ebca4a
 )
diff --git a/go.sum b/go.sum
index 14d5735d6..bd3e44f06 100644
--- a/go.sum
+++ b/go.sum
@@ -515,6 +515,8 @@ gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.7 h1:VUgggvou5XRW9mHwD/yXxIYSMtY0zoKQf/v226p2nyo=
 gopkg.in/yaml.v2 v2.2.7/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v3 v3.0.0-20200121175148-a6ecf24a6d71 h1:Xe2gvTZUJpsvOWUnvmL/tmhVBZUmHSvLbMjRj6NUUKo=
+gopkg.in/yaml.v3 v3.0.0-20200121175148-a6ecf24a6d71/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
 honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
 honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
diff --git a/internal/cmd/cmd.go b/internal/cmd/cmd.go
index 7d8a8e346..4f25d9523 100644
--- a/internal/cmd/cmd.go
+++ b/internal/cmd/cmd.go
@@ -1,7 +1,6 @@
 package cmd
 
 import (
-	"encoding/json"
 	"fmt"
 	"io"
 	"io/ioutil"
@@ -10,6 +9,7 @@ import (
 	"path/filepath"
 
 	"github.com/spf13/cobra"
+	yaml "gopkg.in/yaml.v3"
 
 	"github.com/kyleconroy/sqlc/internal/config"
 )
@@ -47,16 +47,16 @@ var versionCmd = &cobra.Command{
 
 var initCmd = &cobra.Command{
 	Use:   "init",
-	Short: "Create an empty sqlc.json settings file",
+	Short: "Create an empty sqlc.yaml settings file",
 	RunE: func(cmd *cobra.Command, args []string) error {
-		if _, err := os.Stat("sqlc.json"); !os.IsNotExist(err) {
+		if _, err := os.Stat("sqlc.yaml"); !os.IsNotExist(err) {
 			return nil
 		}
-		blob, err := json.MarshalIndent(config.Config{Version: "1"}, "", "  ")
+		blob, err := yaml.Marshal(config.V1GenerateSettings{Version: "1"})
 		if err != nil {
 			return err
 		}
-		return ioutil.WriteFile("sqlc.json", blob, 0644)
+		return ioutil.WriteFile("sqlc.yaml", blob, 0644)
 	},
 }
 
diff --git a/internal/cmd/generate.go b/internal/cmd/generate.go
index 03a81e387..e4a434bb0 100644
--- a/internal/cmd/generate.go
+++ b/internal/cmd/generate.go
@@ -2,9 +2,11 @@ package cmd
 
 import (
 	"bytes"
+	"errors"
 	"fmt"
 	"io"
 	"io/ioutil"
+	"os"
 	"path/filepath"
 	"strings"
 
@@ -34,7 +36,33 @@ func printFileErr(stderr io.Writer, dir string, fileErr dinosql.FileErr) {
 }
 
 func Generate(dir string, stderr io.Writer) (map[string]string, error) {
-	blob, err := ioutil.ReadFile(filepath.Join(dir, "sqlc.json"))
+	var yamlMissing, jsonMissing bool
+	yamlPath := filepath.Join(dir, "sqlc.yaml")
+	jsonPath := filepath.Join(dir, "sqlc.json")
+
+	if _, err := os.Stat(yamlPath); os.IsNotExist(err) {
+		yamlMissing = true
+	}
+	if _, err := os.Stat(jsonPath); os.IsNotExist(err) {
+		jsonMissing = true
+	}
+
+	if yamlMissing && jsonMissing {
+		fmt.Fprintln(stderr, "error parsing sqlc.json: file does not exist")
+		return nil, errors.New("config file missing")
+	}
+
+	if !yamlMissing && !jsonMissing {
+		fmt.Fprintln(stderr, "error parsing sqlc.json: both files present")
+		return nil, errors.New("sqlc.json and sqlc.yaml present")
+	}
+
+	configPath := yamlPath
+	if yamlMissing {
+		configPath = jsonPath
+	}
+
+	blob, err := ioutil.ReadFile(configPath)
 	if err != nil {
 		fmt.Fprintln(stderr, "error parsing sqlc.json: file does not exist")
 		return nil, err
diff --git a/internal/config/config.go b/internal/config/config.go
index d61c02b9b..ac9c59801 100644
--- a/internal/config/config.go
+++ b/internal/config/config.go
@@ -2,7 +2,6 @@ package config
 
 import (
 	"bytes"
-	"encoding/json"
 	"errors"
 	"fmt"
 	"go/types"
@@ -11,6 +10,8 @@ import (
 	"strings"
 
 	"github.com/kyleconroy/sqlc/internal/pg"
+
+	yaml "gopkg.in/yaml.v3"
 )
 
 const errMessageNoVersion = `The configuration file must have a version number.
@@ -29,7 +30,7 @@ The only supported version is "1".
 const errMessageNoPackages = `No packages are configured`
 
 type versionSetting struct {
-	Number string `json:"version"`
+	Number string `json:"version" yaml:"version"`
 }
 
 type Engine string
@@ -40,57 +41,57 @@ const (
 )
 
 type Config struct {
-	Version string `json:"version"`
-	SQL     []SQL  `json:"sql"`
-	Gen     Gen    `json:"overrides,omitempty"`
+	Version string `json:"version" yaml:"version"`
+	SQL     []SQL  `json:"sql" yaml:"sql"`
+	Gen     Gen    `json:"overrides,omitempty" yaml:"overrides"`
 }
 
 type Gen struct {
-	Go *GenGo `json:"go,omitempty"`
+	Go *GenGo `json:"go,omitempty" yaml:"go"`
 }
 
 type GenGo struct {
-	Overrides []Override        `json:"overrides,omitempty"`
-	Rename    map[string]string `json:"rename,omitempty"`
+	Overrides []Override        `json:"overrides,omitempty" yaml:"overrides"`
+	Rename    map[string]string `json:"rename,omitempty" yaml:"rename"`
 }
 
 type SQL struct {
-	Engine  Engine `json:"engine,omitempty"`
-	Schema  string `json:"schema"`
-	Queries string `json:"queries"`
-	Gen     SQLGen `json:"gen"`
+	Engine  Engine `json:"engine,omitempty" yaml:"engine"`
+	Schema  string `json:"schema" yaml:"schema"`
+	Queries string `json:"queries" yaml:"queries"`
+	Gen     SQLGen `json:"gen" yaml:"gen"`
 }
 
 type SQLGen struct {
-	Go *SQLGo `json:"go,omitempty"`
+	Go *SQLGo `json:"go,omitempty" yaml:"go"`
 }
 
 type SQLGo struct {
-	EmitInterface       bool              `json:"emit_interface"`
-	EmitJSONTags        bool              `json:"emit_json_tags"`
-	EmitPreparedQueries bool              `json:"emit_prepared_queries"`
-	Package             string            `json:"package"`
-	Out                 string            `json:"out"`
-	Overrides           []Override        `json:"overrides,omitempty"`
-	Rename              map[string]string `json:"rename,omitempty"`
+	EmitInterface       bool              `json:"emit_interface" yaml:"emit_interface"`
+	EmitJSONTags        bool              `json:"emit_json_tags" yaml:"emit_json_tags"`
+	EmitPreparedQueries bool              `json:"emit_prepared_queries" yaml:"emit_prepared_queries":`
+	Package             string            `json:"package" yaml:"package"`
+	Out                 string            `json:"out" yaml:"out"`
+	Overrides           []Override        `json:"overrides,omitempty" yaml:"overrides"`
+	Rename              map[string]string `json:"rename,omitempty" yaml:"rename"`
 }
 
 type Override struct {
 	// name of the golang type to use, e.g. `github.com/segmentio/ksuid.KSUID`
-	GoType string `json:"go_type"`
+	GoType string `json:"go_type" yaml:"go_type"`
 
 	// fully qualified name of the Go type, e.g. `github.com/segmentio/ksuid.KSUID`
-	DBType                  string `json:"db_type"`
-	Deprecated_PostgresType string `json:"postgres_type"`
+	DBType                  string `json:"db_type" yaml:"db_type"`
+	Deprecated_PostgresType string `json:"postgres_type" yaml:"postgres_type"`
 
 	// for global overrides only when two different engines are in use
-	Engine Engine `json:"engine,omitempty"`
+	Engine Engine `json:"engine,omitempty" yaml:"engine"`
 
 	// True if the GoType should override if the maching postgres type is nullable
-	Null bool `json:"null"`
+	Null bool `json:"null" yaml:"null"`
 
 	// fully qualified name of the column, e.g. `accounts.id`
-	Column string `json:"column"`
+	Column string `json:"column" yaml:"column"`
 
 	ColumnName  string
 	Table       pg.FQN
@@ -202,8 +203,9 @@ func ParseConfig(rd io.Reader) (Config, error) {
 	var buf bytes.Buffer
 	var config Config
 	var version versionSetting
+
 	ver := io.TeeReader(rd, &buf)
-	dec := json.NewDecoder(ver)
+	dec := yaml.NewDecoder(ver)
 	if err := dec.Decode(&version); err != nil {
 		return config, err
 	}
diff --git a/internal/config/v_one.go b/internal/config/v_one.go
index 29d3deb5f..082691439 100644
--- a/internal/config/v_one.go
+++ b/internal/config/v_one.go
@@ -1,35 +1,36 @@
 package config
 
 import (
-	"encoding/json"
 	"fmt"
 	"io"
 	"path/filepath"
+
+	yaml "gopkg.in/yaml.v3"
 )
 
-type v1GenerateSettings struct {
-	Version   string              `json:"version"`
-	Packages  []v1PackageSettings `json:"packages"`
-	Overrides []Override          `json:"overrides,omitempty"`
-	Rename    map[string]string   `json:"rename,omitempty"`
+type V1GenerateSettings struct {
+	Version   string              `json:"version" yaml:"version"`
+	Packages  []v1PackageSettings `json:"packages" yaml:"packages"`
+	Overrides []Override          `json:"overrides,omitempty" yaml:"overrides,omitempty"`
+	Rename    map[string]string   `json:"rename,omitempty" yaml:"rename,omitempty"`
 }
 
 type v1PackageSettings struct {
-	Name                string     `json:"name"`
-	Engine              Engine     `json:"engine,omitempty"`
-	Path                string     `json:"path"`
-	Schema              string     `json:"schema"`
-	Queries             string     `json:"queries"`
-	EmitInterface       bool       `json:"emit_interface"`
-	EmitJSONTags        bool       `json:"emit_json_tags"`
-	EmitPreparedQueries bool       `json:"emit_prepared_queries"`
-	Overrides           []Override `json:"overrides"`
+	Name                string     `json:"name" yaml:"name"`
+	Engine              Engine     `json:"engine,omitempty" yaml:"engine"`
+	Path                string     `json:"path" yaml:"path"`
+	Schema              string     `json:"schema" yaml:"schema"`
+	Queries             string     `json:"queries" yaml:"queries"`
+	EmitInterface       bool       `json:"emit_interface" yaml:"emit_interface"`
+	EmitJSONTags        bool       `json:"emit_json_tags" yaml:"emit_json_tags"`
+	EmitPreparedQueries bool       `json:"emit_prepared_queries" yaml:"emit_prepared_queries"`
+	Overrides           []Override `json:"overrides" yaml:"overrides"`
 }
 
 func v1ParseConfig(rd io.Reader) (Config, error) {
-	dec := json.NewDecoder(rd)
-	dec.DisallowUnknownFields()
-	var settings v1GenerateSettings
+	dec := yaml.NewDecoder(rd)
+	dec.KnownFields(true)
+	var settings V1GenerateSettings
 	var config Config
 	if err := dec.Decode(&settings); err != nil {
 		return config, err
@@ -70,7 +71,7 @@ func v1ParseConfig(rd io.Reader) (Config, error) {
 	return settings.Translate(), nil
 }
 
-func (c *v1GenerateSettings) ValidateGlobalOverrides() error {
+func (c *V1GenerateSettings) ValidateGlobalOverrides() error {
 	engines := map[Engine]struct{}{}
 	for _, pkg := range c.Packages {
 		if _, ok := engines[pkg.Engine]; !ok {
@@ -87,7 +88,7 @@ func (c *v1GenerateSettings) ValidateGlobalOverrides() error {
 	return nil
 }
 
-func (c *v1GenerateSettings) Translate() Config {
+func (c *V1GenerateSettings) Translate() Config {
 	conf := Config{
 		Version: c.Version,
 	}
diff --git a/internal/config/v_two.go b/internal/config/v_two.go
index 78ff05984..4d795016b 100644
--- a/internal/config/v_two.go
+++ b/internal/config/v_two.go
@@ -1,15 +1,16 @@
 package config
 
 import (
-	"encoding/json"
 	"fmt"
 	"io"
 	"path/filepath"
+
+	yaml "gopkg.in/yaml.v3"
 )
 
 func v2ParseConfig(rd io.Reader) (Config, error) {
-	dec := json.NewDecoder(rd)
-	dec.DisallowUnknownFields()
+	dec := yaml.NewDecoder(rd)
+	dec.KnownFields(true)
 	var conf Config
 	if err := dec.Decode(&conf); err != nil {
 		return conf, err


Test patch:
diff --git a/internal/config/config_test.go b/internal/config/config_test.go
index da09091c3..ef12a178b 100644
--- a/internal/config/config_test.go
+++ b/internal/config/config_test.go
@@ -46,7 +46,8 @@ func TestBadConfigs(t *testing.T) {
 		},
 		{
 			"unknown fields",
-			"json: unknown field \"foo\"",
+			`yaml: unmarshal errors:
+  line 3: field foo not found in type config.V1GenerateSettings`,
 			unknownFields,
 		},
 	} {
diff --git a/internal/endtoend/testdata/yaml_overrides/go/db.go b/internal/endtoend/testdata/yaml_overrides/go/db.go
new file mode 100644
index 000000000..6f21c94ed
--- /dev/null
+++ b/internal/endtoend/testdata/yaml_overrides/go/db.go
@@ -0,0 +1,29 @@
+// Code generated by sqlc. DO NOT EDIT.
+
+package override
+
+import (
+	"context"
+	"database/sql"
+)
+
+type DBTX interface {
+	ExecContext(context.Context, string, ...interface{}) (sql.Result, error)
+	PrepareContext(context.Context, string) (*sql.Stmt, error)
+	QueryContext(context.Context, string, ...interface{}) (*sql.Rows, error)
+	QueryRowContext(context.Context, string, ...interface{}) *sql.Row
+}
+
+func New(db DBTX) *Queries {
+	return &Queries{db: db}
+}
+
+type Queries struct {
+	db DBTX
+}
+
+func (q *Queries) WithTx(tx *sql.Tx) *Queries {
+	return &Queries{
+		db: tx,
+	}
+}
diff --git a/internal/endtoend/testdata/yaml_overrides/go/models.go b/internal/endtoend/testdata/yaml_overrides/go/models.go
new file mode 100644
index 000000000..fb9384688
--- /dev/null
+++ b/internal/endtoend/testdata/yaml_overrides/go/models.go
@@ -0,0 +1,17 @@
+// Code generated by sqlc. DO NOT EDIT.
+
+package override
+
+import (
+	"example.com/pkg"
+	"github.com/lib/pq"
+)
+
+type Foo struct {
+	Other   string
+	Total   int64
+	Tags    []string
+	ByteSeq []byte
+	Retyped pkg.CustomType
+	Langs   pq.StringArray
+}
diff --git a/internal/endtoend/testdata/yaml_overrides/sql/query.sql b/internal/endtoend/testdata/yaml_overrides/sql/query.sql
new file mode 100644
index 000000000..e0ac49d1e
--- /dev/null
+++ b/internal/endtoend/testdata/yaml_overrides/sql/query.sql
@@ -0,0 +1,1 @@
+SELECT 1;
diff --git a/internal/endtoend/testdata/yaml_overrides/sql/schema.sql b/internal/endtoend/testdata/yaml_overrides/sql/schema.sql
new file mode 100644
index 000000000..fc3a13e53
--- /dev/null
+++ b/internal/endtoend/testdata/yaml_overrides/sql/schema.sql
@@ -0,0 +1,8 @@
+CREATE TABLE foo (
+    other     text NOT NULL,
+    total     bigint NOT NULL,
+    tags      text[] NOT NULL,
+    byte_seq  bytea NOT NULL,
+    retyped   text NOT NULL,
+    langs     text[]
+);
diff --git a/internal/endtoend/testdata/yaml_overrides/sqlc.yaml b/internal/endtoend/testdata/yaml_overrides/sqlc.yaml
new file mode 100644
index 000000000..d91ace123
--- /dev/null
+++ b/internal/endtoend/testdata/yaml_overrides/sqlc.yaml
@@ -0,0 +1,11 @@
+version: 1
+packages:
+  - path: "go"
+    name: "override"
+    schema: "sql/"
+    queries: "sql/"
+    overrides:
+      - go_type: "example.com/pkg.CustomType"
+        column: "foo.retyped"
+      - go_type: "github.com/lib/pq.StringArray"
+        column: "foo.langs"