Repository: kyleconroy/sqlc
Pull request #1436 (2022-02-13 05:05:12)

feat: Upload projects
- [x] Add documentation for `sqlc upload`
- [x] Add reference documentation for the upload endpoint


Patch:
diff --git a/docs/guides/privacy.md b/docs/guides/privacy.md
index 1422fea99..da29fd885 100644
--- a/docs/guides/privacy.md
+++ b/docs/guides/privacy.md
@@ -49,7 +49,7 @@ We provide a few hosted services in addition to the sqlc command line tool.
 * Playground data stored in [Google Cloud Storage](https://cloud.google.com/storage)
   * Automatically deleted after 30 days
 
-### api.sqlc.dev
+### app.sqlc.dev / api.sqlc.dev
 
 * Hosted on [Heroku](https://heroku.com)
 * Error tracking and tracing with [Sentry](https://sentry.io)
diff --git a/docs/howto/upload.md b/docs/howto/upload.md
new file mode 100644
index 000000000..0add35c94
--- /dev/null
+++ b/docs/howto/upload.md
@@ -0,0 +1,57 @@
+# Uploading projects
+
+*This feature requires signing up for [sqlc Cloud](https://app.sqlc.dev), which is currently in beta.*
+
+Uploading your project ensures that future releases of sqlc do not break your
+existing code. Similar to Rust's [crater](https://github.com/rust-lang/crater)
+project, uploaded projects are tested against development releases of sqlc to
+verify correctness.
+
+## Add configuration
+
+After creating a project, add the project ID to your sqlc configuration file.
+
+```yaml
+version: "1"
+project:
+  id: "<PROJECT-ID>"
+packages: []
+```
+
+```json
+{
+  "version": "1",
+  "project": {
+    "id": "<PROJECT-ID>"
+  },
+  "packages": [
+  ]
+}
+```
+
+You'll also need to create an API token and make it available via the
+`SQLC_AUTH_TOKEN` environment variable.
+
+```shell
+export SQLC_AUTH_TOKEN=sqlc_xxxxxxxx
+```
+
+## Dry run
+
+You can see what's included when uploading your project by using using the `--dry-run` flag:
+
+```shell
+sqlc upload --dry-run
+```
+
+The output will be the exact HTTP request sent by `sqlc`.
+
+## Upload
+
+Once you're ready to upload, remove the `--dry-run` flag.
+
+```shell
+sqlc upload
+```
+
+By uploading your project, you're making sqlc more stable and reliable. Thanks!
diff --git a/docs/index.rst b/docs/index.rst
index e0f39532f..91b68c02f 100644
--- a/docs/index.rst
+++ b/docs/index.rst
@@ -53,6 +53,8 @@ code ever again.
    howto/ddl.md
    howto/structs.md
 
+   howto/upload.md
+
 .. toctree::
    :maxdepth: 2
    :caption: Reference
diff --git a/docs/reference/cli.md b/docs/reference/cli.md
index ec55f3b9e..a6033f220 100644
--- a/docs/reference/cli.md
+++ b/docs/reference/cli.md
@@ -6,13 +6,17 @@ Usage:
 
 Available Commands:
   compile     Statically check SQL for syntax and type errors
+  completion  Generate the autocompletion script for the specified shell
   generate    Generate Go code from SQL
   help        Help about any command
   init        Create an empty sqlc.yaml settings file
+  upload      Upload the schema, queries, and configuration for this project
   version     Print the sqlc version number
 
 Flags:
-  -h, --help   help for sqlc
+  -x, --experimental   enable experimental features (default: false)
+  -f, --file string    specify an alternate config file (default: sqlc.yaml)
+  -h, --help           help for sqlc
 
 Use "sqlc [command] --help" for more information about a command.
 ```
diff --git a/internal/bundler/metadata.go b/internal/bundler/metadata.go
new file mode 100644
index 000000000..415b131ef
--- /dev/null
+++ b/internal/bundler/metadata.go
@@ -0,0 +1,16 @@
+package bundler
+
+import (
+	"runtime"
+
+	"github.com/kyleconroy/sqlc/internal/info"
+)
+
+func projectMetadata() ([][2]string, error) {
+	return [][2]string{
+		{"sqlc_version", info.Version},
+		{"go_version", runtime.Version()},
+		{"goos", runtime.GOOS},
+		{"goarch", runtime.GOARCH},
+	}, nil
+}
diff --git a/internal/bundler/multipart.go b/internal/bundler/multipart.go
new file mode 100644
index 000000000..a755bbfcf
--- /dev/null
+++ b/internal/bundler/multipart.go
@@ -0,0 +1,80 @@
+package bundler
+
+import (
+	"io"
+	"mime/multipart"
+	"os"
+	"path/filepath"
+
+	"github.com/kyleconroy/sqlc/internal/config"
+	"github.com/kyleconroy/sqlc/internal/sql/sqlpath"
+)
+
+func writeInputs(w *multipart.Writer, file string, conf *config.Config) error {
+	refs := map[string]struct{}{}
+	refs[filepath.Base(file)] = struct{}{}
+
+	for _, pkg := range conf.SQL {
+		for _, paths := range []config.Paths{pkg.Schema, pkg.Queries} {
+			files, err := sqlpath.Glob(paths)
+			if err != nil {
+				return err
+			}
+			for _, file := range files {
+				refs[file] = struct{}{}
+			}
+		}
+	}
+
+	for file, _ := range refs {
+		if err := addPart(w, file); err != nil {
+			return err
+		}
+	}
+
+	params, err := projectMetadata()
+	if err != nil {
+		return err
+	}
+	params = append(params, [2]string{"project_id", conf.Project.ID})
+	for _, val := range params {
+		if err = w.WriteField(val[0], val[1]); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func addPart(w *multipart.Writer, file string) error {
+	h, err := os.Open(file)
+	if err != nil {
+		return err
+	}
+	defer h.Close()
+	part, err := w.CreateFormFile("inputs", file)
+	if err != nil {
+		return err
+	}
+	_, err = io.Copy(part, h)
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+func writeOutputs(w *multipart.Writer, dir string, output map[string]string) error {
+	for filename, contents := range output {
+		rel, err := filepath.Rel(dir, filename)
+		if err != nil {
+			return err
+		}
+		part, err := w.CreateFormFile("outputs", rel)
+		if err != nil {
+			return err
+		}
+		if _, err := io.WriteString(part, contents); err != nil {
+			return err
+		}
+	}
+	return nil
+}
diff --git a/internal/bundler/upload.go b/internal/bundler/upload.go
new file mode 100644
index 000000000..8f09f8562
--- /dev/null
+++ b/internal/bundler/upload.go
@@ -0,0 +1,101 @@
+package bundler
+
+import (
+	"bytes"
+	"context"
+	"fmt"
+	"io"
+	"mime/multipart"
+	"net/http"
+	"net/http/httputil"
+	"os"
+
+	"github.com/kyleconroy/sqlc/internal/config"
+)
+
+type Uploader struct {
+	token      string
+	configPath string
+	config     *config.Config
+	dir        string
+}
+
+func NewUploader(configPath, dir string, conf *config.Config) *Uploader {
+	return &Uploader{
+		token:      os.Getenv("SQLC_AUTH_TOKEN"),
+		configPath: configPath,
+		config:     conf,
+		dir:        dir,
+	}
+}
+
+func (up *Uploader) Validate() error {
+	if up.config.Project.ID == "" {
+		return fmt.Errorf("project.id is not set")
+	}
+	if up.token == "" {
+		return fmt.Errorf("SQLC_AUTH_TOKEN environment variable is not set")
+	}
+	return nil
+}
+
+func (up *Uploader) buildRequest(ctx context.Context, result map[string]string) (*http.Request, error) {
+	body := bytes.NewBuffer([]byte{})
+
+	w := multipart.NewWriter(body)
+	defer w.Close()
+	if err := writeInputs(w, up.configPath, up.config); err != nil {
+		return nil, err
+	}
+	if err := writeOutputs(w, up.dir, result); err != nil {
+		return nil, err
+	}
+	w.Close()
+
+	req, err := http.NewRequest("POST", "https://api.sqlc.dev/upload", body)
+	if err != nil {
+		return nil, err
+	}
+
+	// Set sqlc-version header
+	req.Header.Set("Content-Type", w.FormDataContentType())
+	req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", up.token))
+	return req.WithContext(ctx), nil
+}
+
+func (up *Uploader) DumpRequestOut(ctx context.Context, result map[string]string) error {
+	req, err := up.buildRequest(ctx, result)
+	if err != nil {
+		return err
+	}
+	dump, err := httputil.DumpRequest(req, true)
+	if err != nil {
+		return err
+	}
+	os.Stdout.Write(dump)
+	return nil
+}
+
+func (up *Uploader) Upload(ctx context.Context, result map[string]string) error {
+	if err := up.Validate(); err != nil {
+		return err
+	}
+	req, err := up.buildRequest(ctx, result)
+	if err != nil {
+		return err
+	}
+	client := &http.Client{}
+	resp, err := client.Do(req)
+	if err != nil {
+		return err
+	}
+	if resp.StatusCode >= 400 {
+		body, err := io.ReadAll(resp.Body)
+		defer resp.Body.Close()
+		if err != nil {
+			return fmt.Errorf("upload error: endpoint returned non-200 status code: %d", resp.StatusCode)
+		}
+		return fmt.Errorf("upload error: %d: %s", resp.StatusCode, string(body))
+	}
+	return nil
+}
diff --git a/internal/cmd/cmd.go b/internal/cmd/cmd.go
index 5cefe6a64..4c36e96b3 100644
--- a/internal/cmd/cmd.go
+++ b/internal/cmd/cmd.go
@@ -15,6 +15,7 @@ import (
 
 	"github.com/kyleconroy/sqlc/internal/config"
 	"github.com/kyleconroy/sqlc/internal/debug"
+	"github.com/kyleconroy/sqlc/internal/info"
 	"github.com/kyleconroy/sqlc/internal/tracer"
 )
 
@@ -28,6 +29,8 @@ func Do(args []string, stdin io.Reader, stdout io.Writer, stderr io.Writer) int
 	rootCmd.AddCommand(genCmd)
 	rootCmd.AddCommand(initCmd)
 	rootCmd.AddCommand(versionCmd)
+	uploadCmd.Flags().BoolP("dry-run", "", false, "dump upload request (default: false)")
+	rootCmd.AddCommand(uploadCmd)
 
 	rootCmd.SetArgs(args)
 	rootCmd.SetIn(stdin)
@@ -60,9 +63,7 @@ var versionCmd = &cobra.Command{
 			defer trace.StartRegion(cmd.Context(), "version").End()
 		}
 		if version == "" {
-			// When no version is set, return the next bug fix version
-			// after the most recent tag
-			fmt.Printf("%s\n", "v1.12.0")
+			fmt.Printf("%s\n", info.Version)
 		} else {
 			fmt.Printf("%s\n", version)
 		}
@@ -96,11 +97,16 @@ var initCmd = &cobra.Command{
 
 type Env struct {
 	ExperimentalFeatures bool
+	DryRun               bool
 }
 
 func ParseEnv(c *cobra.Command) Env {
 	x := c.Flag("experimental")
-	return Env{ExperimentalFeatures: x != nil && x.Changed}
+	dr := c.Flag("dry-run")
+	return Env{
+		ExperimentalFeatures: x != nil && x.Changed,
+		DryRun:               dr != nil && dr.Changed,
+	}
 }
 
 func getConfigPath(stderr io.Writer, f *pflag.Flag) (string, string) {
@@ -152,6 +158,20 @@ var genCmd = &cobra.Command{
 	},
 }
 
+var uploadCmd = &cobra.Command{
+	Use:   "upload",
+	Short: "Upload the schema, queries, and configuration for this project",
+	RunE: func(cmd *cobra.Command, args []string) error {
+		stderr := cmd.ErrOrStderr()
+		dir, name := getConfigPath(stderr, cmd.Flag("file"))
+		if err := createPkg(cmd.Context(), ParseEnv(cmd), dir, name, stderr); err != nil {
+			fmt.Fprintf(stderr, "error uploading: %s\n", err)
+			os.Exit(1)
+		}
+		return nil
+	},
+}
+
 var checkCmd = &cobra.Command{
 	Use:   "compile",
 	Short: "Statically check SQL for syntax and type errors",
diff --git a/internal/cmd/generate.go b/internal/cmd/generate.go
index c25d37bc0..9d53cd4f9 100644
--- a/internal/cmd/generate.go
+++ b/internal/cmd/generate.go
@@ -47,7 +47,7 @@ type outPair struct {
 	config.SQL
 }
 
-func Generate(ctx context.Context, e Env, dir, filename string, stderr io.Writer) (map[string]string, error) {
+func readConfig(stderr io.Writer, dir, filename string) (string, *config.Config, error) {
 	configPath := ""
 	if filename != "" {
 		configPath = filepath.Join(dir, filename)
@@ -65,12 +65,12 @@ func Generate(ctx context.Context, e Env, dir, filename string, stderr io.Writer
 
 		if yamlMissing && jsonMissing {
 			fmt.Fprintln(stderr, "error parsing configuration files. sqlc.yaml or sqlc.json: file does not exist")
-			return nil, errors.New("config file missing")
+			return "", nil, errors.New("config file missing")
 		}
 
 		if !yamlMissing && !jsonMissing {
 			fmt.Fprintln(stderr, "error: both sqlc.json and sqlc.yaml files present")
-			return nil, errors.New("sqlc.json and sqlc.yaml present")
+			return "", nil, errors.New("sqlc.json and sqlc.yaml present")
 		}
 
 		configPath = yamlPath
@@ -83,7 +83,7 @@ func Generate(ctx context.Context, e Env, dir, filename string, stderr io.Writer
 	blob, err := os.ReadFile(configPath)
 	if err != nil {
 		fmt.Fprintf(stderr, "error parsing %s: file does not exist\n", base)
-		return nil, err
+		return "", nil, err
 	}
 
 	conf, err := config.ParseConfig(bytes.NewReader(blob))
@@ -97,9 +97,19 @@ func Generate(ctx context.Context, e Env, dir, filename string, stderr io.Writer
 			fmt.Fprintf(stderr, errMessageNoPackages)
 		}
 		fmt.Fprintf(stderr, "error parsing %s: %s\n", base, err)
+		return "", nil, err
+	}
+
+	return configPath, &conf, nil
+}
+
+func Generate(ctx context.Context, e Env, dir, filename string, stderr io.Writer) (map[string]string, error) {
+	configPath, conf, err := readConfig(stderr, dir, filename)
+	if err != nil {
 		return nil, err
 	}
 
+	base := filepath.Base(configPath)
 	if err := config.Validate(conf); err != nil {
 		fmt.Fprintf(stderr, "error validating %s: %s\n", base, err)
 		return nil, err
@@ -135,7 +145,7 @@ func Generate(ctx context.Context, e Env, dir, filename string, stderr io.Writer
 	}
 
 	for _, sql := range pairs {
-		combo := config.Combine(conf, sql.SQL)
+		combo := config.Combine(*conf, sql.SQL)
 
 		// TODO: This feels like a hack that will bite us later
 		joined := make([]string, 0, len(sql.Schema))
diff --git a/internal/cmd/package.go b/internal/cmd/package.go
new file mode 100644
index 000000000..63a4e0050
--- /dev/null
+++ b/internal/cmd/package.go
@@ -0,0 +1,29 @@
+package cmd
+
+import (
+	"context"
+	"io"
+	"os"
+
+	"github.com/kyleconroy/sqlc/internal/bundler"
+)
+
+func createPkg(ctx context.Context, e Env, dir, filename string, stderr io.Writer) error {
+	configPath, conf, err := readConfig(stderr, dir, filename)
+	if err != nil {
+		return err
+	}
+	up := bundler.NewUploader(configPath, dir, conf)
+	if err := up.Validate(); err != nil {
+		return err
+	}
+	output, err := Generate(ctx, e, dir, filename, stderr)
+	if err != nil {
+		os.Exit(1)
+	}
+	if e.DryRun {
+		return up.DumpRequestOut(ctx, output)
+	} else {
+		return up.Upload(ctx, output)
+	}
+}
diff --git a/internal/config/config.go b/internal/config/config.go
index 66ba3f261..0cc641925 100644
--- a/internal/config/config.go
+++ b/internal/config/config.go
@@ -78,9 +78,14 @@ const (
 )
 
 type Config struct {
-	Version string `json:"version" yaml:"version"`
-	SQL     []SQL  `json:"sql" yaml:"sql"`
-	Gen     Gen    `json:"overrides,omitempty" yaml:"overrides"`
+	Version string  `json:"version" yaml:"version"`
+	Project Project `json:"project" yaml:"project"`
+	SQL     []SQL   `json:"sql" yaml:"sql"`
+	Gen     Gen     `json:"overrides,omitempty" yaml:"overrides"`
+}
+
+type Project struct {
+	ID string `json:"id" yaml:"id"`
 }
 
 type Gen struct {
@@ -327,7 +332,7 @@ func ParseConfig(rd io.Reader) (Config, error) {
 	}
 }
 
-func Validate(c Config) error {
+func Validate(c *Config) error {
 	for _, sql := range c.SQL {
 		sqlGo := sql.Gen.Go
 		if sqlGo == nil {
diff --git a/internal/config/v_one.go b/internal/config/v_one.go
index d1ad4fc11..5b357255b 100644
--- a/internal/config/v_one.go
+++ b/internal/config/v_one.go
@@ -10,6 +10,7 @@ import (
 
 type V1GenerateSettings struct {
 	Version   string              `json:"version" yaml:"version"`
+	Project   Project             `json:"project" yaml:"project"`
 	Packages  []v1PackageSettings `json:"packages" yaml:"packages"`
 	Overrides []Override          `json:"overrides,omitempty" yaml:"overrides,omitempty"`
 	Rename    map[string]string   `json:"rename,omitempty" yaml:"rename,omitempty"`
@@ -105,6 +106,7 @@ func (c *V1GenerateSettings) ValidateGlobalOverrides() error {
 func (c *V1GenerateSettings) Translate() Config {
 	conf := Config{
 		Version: c.Version,
+		Project: c.Project,
 	}
 
 	for _, pkg := range c.Packages {
diff --git a/internal/info/facts.go b/internal/info/facts.go
new file mode 100644
index 000000000..3c4b47849
--- /dev/null
+++ b/internal/info/facts.go
@@ -0,0 +1,5 @@
+package info
+
+// When no version is set, return the next bug fix version
+// after the most recent tag
+const Version = "v1.12.0"


Test patch:
diff --git a/internal/config/config_test.go b/internal/config/config_test.go
index 9250629c2..183549ea5 100644
--- a/internal/config/config_test.go
+++ b/internal/config/config_test.go
@@ -65,7 +65,7 @@ func TestBadConfigs(t *testing.T) {
 }
 
 func TestInvalidConfig(t *testing.T) {
-	err := Validate(Config{
+	err := Validate(&Config{
 		SQL: []SQL{{
 			Gen: SQLGen{
 				Go: &SQLGo{