Repository: hamba/avro
Pull request #260 (2023-06-02 17:41:33)

feat: implement simple Avro schema validation utility
Hi !
First of all, thank you for this library. It is really useful and well designed/implemented. Kudos!

I wanted to contribute back a small command-line utility I developed using this library that turned out very useful for us. The tool allows to validate Avro schemas, i.e. check that they are well-formed/parseable. We use this tool mostly in our CI/CD pipelines as an extra layer of protection to schema syntax mistakes/breaking.

I based the code pretty much on the existing `avrogen` utility code for consistency and also added unit tests.
Note that I had to manually clear the global schemas cache after each subtest in `TestAvroSv_ValidatesSchema`, otherwise a previous subtest state influences the next one.

Given that I was there, I also fixed an existing minor typo in the README (`./app` instead of `avrogen`).

I hope you are okay with this contribution, if you prefer not to accept it, no hard feelings!

nrwiersma: Thanks for the contribution 🎉  This seems like a simple but valuable addition.

Patch:
diff --git a/README.md b/README.md
index 6ab1692..433683d 100644
--- a/README.md
+++ b/README.md
@@ -164,7 +164,7 @@ go install github.com/hamba/avro/v2/cmd/avrogen@<version>
 Example usage assuming there's a valid schema in `in.avsc`:
 
 ```shell
-./app -pkg avro -o bla.go -tags json:snake,yaml:upper-camel in.avsc
+avrogen -pkg avro -o bla.go -tags json:snake,yaml:upper-camel in.avsc
 ```
 
 Check the options and usage with `-h`:
@@ -175,6 +175,44 @@ avrogen -h
 
 Or use it as a lib in internal commands, it's the `gen` package
 
+## Avro schema validation
+
+A small Avro schema validation command-line utility is also available. This simple tool leverages the
+schema parsing functionality of the library, showing validation errors or optionally dumping parsed
+schemas to the console. It can be used in CI/CD pipelines to validate schema changes in a repository.
+
+Install the Avro schema validator with:
+
+```shell
+go install github.com/hamba/avro/v2/cmd/avrosv@<version>
+```
+
+Example usage assuming there's a valid schema in `in.avsc` (exit status code is `0`):
+
+```shell
+avrosv in.avsc
+```
+
+An invalid schema will result in a diagnostic output and a non-zero exit status code:
+
+```shell
+avrosv bad-default-schema.avsc; echo $?
+Error: avro: invalid default for field someString. <nil> not a string
+2
+```
+
+Schemas referencing other schemas can also be validated by providing all of them (schemas are parsed in order):
+
+```shell
+avrosv base-schema.avsc schema-withref.avsc
+```
+
+Check the options and usage with `-h`:
+
+```shell
+avrosv -h
+```
+
 ## Go Version Support
 
 This library supports the last two versions of Go. While the minimum Go version is
diff --git a/cmd/avrosv/main.go b/cmd/avrosv/main.go
new file mode 100644
index 0000000..26f737b
--- /dev/null
+++ b/cmd/avrosv/main.go
@@ -0,0 +1,50 @@
+package main
+
+import (
+	"flag"
+	"fmt"
+	"io"
+	"os"
+
+	"github.com/hamba/avro/v2"
+)
+
+type config struct {
+	Verbose bool
+}
+
+func main() {
+	os.Exit(realMain(os.Args, os.Stderr, os.Stdout))
+}
+
+func realMain(args []string, out, dumpout io.Writer) int {
+	var cfg config
+	flgs := flag.NewFlagSet("avrosv", flag.ExitOnError)
+	flgs.SetOutput(out)
+	flgs.BoolVar(&cfg.Verbose, "v", false, "Verbose output (dump final parsed schema).")
+	flgs.Usage = func() {
+		_, _ = fmt.Fprintln(out, "Usage: avrosv [options] schemas")
+		_, _ = fmt.Fprintln(out, "Options:")
+		flgs.PrintDefaults()
+		_, _ = fmt.Fprintln(out, "\nSchemas are processed in the order they appear.")
+	}
+	if err := flgs.Parse(args[1:]); err != nil {
+		return 1
+	}
+	if flgs.NArg() < 1 {
+		_, _ = fmt.Fprintln(out, "Error: at least one schema is required")
+		return 1
+	}
+
+	schema, err := avro.ParseFiles(flgs.Args()...)
+	if err != nil {
+		_, _ = fmt.Fprintf(out, "Error: %v\n", err)
+		return 2
+	}
+
+	if cfg.Verbose {
+		fmt.Fprintln(dumpout, schema)
+	}
+
+	return 0
+}


Test patch:
diff --git a/cmd/avrosv/main_test.go b/cmd/avrosv/main_test.go
new file mode 100644
index 0000000..d4634ed
--- /dev/null
+++ b/cmd/avrosv/main_test.go
@@ -0,0 +1,130 @@
+package main
+
+import (
+	"bytes"
+	"io"
+	"testing"
+
+	"github.com/hamba/avro/v2"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestAvroSv_RequiredFlags(t *testing.T) {
+	tests := []struct {
+		name    string
+		args    []string
+		wantErr bool
+	}{
+		{
+			name:    "validates no schema is set",
+			args:    []string{"avrosv"},
+			wantErr: true,
+		},
+		{
+			name:    "validates single schema is set",
+			args:    []string{"avrosv", "some/file"},
+			wantErr: true,
+		},
+		{
+			name:    "validates multiple schemas are set",
+			args:    []string{"avrosv", "some/file", "some/other"},
+			wantErr: true,
+		},
+	}
+
+	for _, test := range tests {
+		test := test
+		t.Run(test.name, func(t *testing.T) {
+			got := realMain(test.args, io.Discard, io.Discard)
+
+			if !test.wantErr {
+				assert.Equal(t, 0, got)
+				return
+			}
+
+			assert.NotEqual(t, 0, got)
+		})
+	}
+}
+
+func TestAvroSv_ValidatesSchema(t *testing.T) {
+	tests := []struct {
+		name    string
+		args    []string
+		wantErr bool
+	}{
+		{
+			name:    "validates a simple schema",
+			args:    []string{"avrosv", "testdata/schema.avsc"},
+			wantErr: false,
+		},
+		{
+			name:    "does not validate a bad schema",
+			args:    []string{"avrosv", "testdata/bad-schema.avsc"},
+			wantErr: true,
+		},
+		{
+			name:    "does not validate a schema with a bad default",
+			args:    []string{"avrosv", "testdata/bad-default-schema.avsc"},
+			wantErr: true,
+		},
+		{
+			name:    "does not validate a schema with a reference to a missing schema",
+			args:    []string{"avrosv", "testdata/withref-schema.avsc"},
+			wantErr: true,
+		},
+		{
+			name:    "validates a schema with a reference to an existing schema",
+			args:    []string{"avrosv", "testdata/schema.avsc", "testdata/withref-schema.avsc"},
+			wantErr: false,
+		},
+	}
+
+	for _, test := range tests {
+		test := test
+		t.Run(test.name, func(t *testing.T) {
+			avro.DefaultSchemaCache = &avro.SchemaCache{} // reset the schema cache
+			got := realMain(test.args, io.Discard, io.Discard)
+
+			if !test.wantErr {
+				assert.Equal(t, 0, got)
+				return
+			}
+
+			assert.NotEqual(t, 0, got)
+		})
+	}
+}
+
+func TestAvroSv_Verbose(t *testing.T) {
+	tests := []struct {
+		name    string
+		args    []string
+		out     string
+		wantErr bool
+	}{
+		{
+			name:    "dumps a simple schema",
+			args:    []string{"avrosv", "-v", "testdata/schema.avsc"},
+			out:     "{\"name\":\"test\",\"type\":\"record\",\"fields\":[{\"name\":\"someString\",\"type\":\"string\"}]}\n",
+			wantErr: false,
+		},
+		{
+			name:    "dumps a schema with a reference to an existing schema",
+			args:    []string{"avrosv", "-v", "testdata/schema.avsc", "testdata/withref-schema.avsc"},
+			out:     "{\"name\":\"testref\",\"type\":\"record\",\"fields\":[{\"name\":\"someref\",\"type\":\"test\"}]}\n",
+			wantErr: false,
+		},
+	}
+
+	for _, test := range tests {
+		test := test
+		t.Run(test.name, func(t *testing.T) {
+			avro.DefaultSchemaCache = &avro.SchemaCache{} // reset the schema cache
+			w := &bytes.Buffer{}
+			got := realMain(test.args, io.Discard, w)
+			assert.Equal(t, 0, got)
+			assert.Equal(t, test.out, w.String())
+		})
+	}
+}
diff --git a/cmd/avrosv/testdata/bad-default-schema.avsc b/cmd/avrosv/testdata/bad-default-schema.avsc
new file mode 100644
index 0000000..5ee1537
--- /dev/null
+++ b/cmd/avrosv/testdata/bad-default-schema.avsc
@@ -0,0 +1,7 @@
+{
+  "type": "record",
+  "name": "test",
+  "fields": [
+    { "name": "someString", "type": "string", "default": null }
+  ]
+}
diff --git a/cmd/avrosv/testdata/bad-schema.avsc b/cmd/avrosv/testdata/bad-schema.avsc
new file mode 100644
index 0000000..0967ef4
--- /dev/null
+++ b/cmd/avrosv/testdata/bad-schema.avsc
@@ -0,0 +1,1 @@
+{}
diff --git a/cmd/avrosv/testdata/schema.avsc b/cmd/avrosv/testdata/schema.avsc
new file mode 100644
index 0000000..ef10edd
--- /dev/null
+++ b/cmd/avrosv/testdata/schema.avsc
@@ -0,0 +1,7 @@
+{
+  "type": "record",
+  "name": "test",
+  "fields": [
+    { "name": "someString", "type": "string" }
+  ]
+}
diff --git a/cmd/avrosv/testdata/withref-schema.avsc b/cmd/avrosv/testdata/withref-schema.avsc
new file mode 100644
index 0000000..1b7482d
--- /dev/null
+++ b/cmd/avrosv/testdata/withref-schema.avsc
@@ -0,0 +1,7 @@
+{
+  "type": "record",
+  "name": "testref",
+  "fields": [
+    { "name": "someref", "type": "test" }
+  ]
+}