Repository: hamba/avro
Pull request #446 (2024-09-09 13:41:34)

feat: use verbose schema for encoders
fixes https://github.com/hamba/avro/issues/441

vdorokhin: > This change breaks backwards compatibility. While this change may suit your needs, it is common for people to use canonical forms of the schema.

With the current implementation, everyone who used the String() method to retrieve the schema will continue receiving the canonical version of the schema. However, if needed, access to other schema data will also be available. In any case, I’m open to making additional changes. I can add a flag to control this behavior. Would that be a suitable option?
nrwiersma: > With the current implementation, everyone who used the String() method to retrieve the schema will continue receiving the canonical version of the schema. However, if needed, access to other schema data will also be available. In any case, I’m open to making additional changes.

While true, you are changing the default behaviour of the application. It is reasonable to assume that the same inputs by enlarge produce the same outputs.

I suggest an option to store the "full" output, perhaps `WithFullSchema`.
vdorokhin: I added the `-fullschema` flag to enable this behavior. In the tests, I used it in combination with `-encoders` for easier verification

Patch:
diff --git a/cmd/avrogen/main.go b/cmd/avrogen/main.go
index a2adfb4..2009101 100644
--- a/cmd/avrogen/main.go
+++ b/cmd/avrogen/main.go
@@ -24,6 +24,7 @@ type config struct {
 	Tags        string
 	FullName    bool
 	Encoders    bool
+	FullSchema  bool
 	StrictTypes bool
 	Initialisms string
 }
@@ -42,6 +43,7 @@ func realMain(args []string, stdout, stderr io.Writer) int {
 	flgs.StringVar(&cfg.Tags, "tags", "", "The additional field tags <tag-name>:{snake|camel|upper-camel|kebab}>[,...]")
 	flgs.BoolVar(&cfg.FullName, "fullname", false, "Use the full name of the Record schema to create the struct name.")
 	flgs.BoolVar(&cfg.Encoders, "encoders", false, "Generate encoders for the structs.")
+	flgs.BoolVar(&cfg.FullSchema, "fullschema", false, "Use the full schema in the generated encoders.")
 	flgs.BoolVar(&cfg.StrictTypes, "strict-types", false, "Use strict type sizes (e.g. int32) during generation.")
 	flgs.StringVar(&cfg.Initialisms, "initialisms", "", "Custom initialisms <VAL>[,...] for struct and field names.")
 	flgs.StringVar(&cfg.TemplateFileName, "template-filename", "", "Override output template with one loaded from file.")
@@ -84,6 +86,7 @@ func realMain(args []string, stdout, stderr io.Writer) int {
 		gen.WithInitialisms(initialisms),
 		gen.WithTemplate(string(template)),
 		gen.WithStrictTypes(cfg.StrictTypes),
+		gen.WithFullSchema(cfg.FullSchema),
 	}
 	g := gen.NewGenerator(cfg.Pkg, tags, opts...)
 	for _, file := range flgs.Args() {
diff --git a/gen/gen.go b/gen/gen.go
index 05e3e1b..010fc85 100644
--- a/gen/gen.go
+++ b/gen/gen.go
@@ -23,6 +23,7 @@ type Config struct {
 	Tags         map[string]TagStyle
 	FullName     bool
 	Encoders     bool
+	FullSchema   bool
 	StrictTypes  bool
 	Initialisms  []string
 	LogicalTypes []LogicalType
@@ -83,6 +84,7 @@ func StructFromSchema(schema avro.Schema, w io.Writer, cfg Config) error {
 		WithEncoders(cfg.Encoders),
 		WithInitialisms(cfg.Initialisms),
 		WithStrictTypes(cfg.StrictTypes),
+		WithFullSchema(cfg.FullSchema),
 	}
 	for _, opt := range cfg.LogicalTypes {
 		opts = append(opts, WithLogicalType(opt))
@@ -159,6 +161,13 @@ func WithPackageDoc(text string) OptsFunc {
 	}
 }
 
+// WithFullSchema configures the generator to store the full schema within the generation context.
+func WithFullSchema(b bool) OptsFunc {
+	return func(g *Generator) {
+		g.fullSchema = b
+	}
+}
+
 // LogicalType used when the name of the "LogicalType" field in the Avro schema matches the Name attribute.
 type LogicalType struct {
 	// Name of the LogicalType
@@ -200,6 +209,7 @@ type Generator struct {
 	tags         map[string]TagStyle
 	fullName     bool
 	encoders     bool
+	fullSchema   bool
 	strictTypes  bool
 	initialisms  []string
 	logicalTypes map[avro.LogicalType]LogicalType
@@ -304,11 +314,22 @@ func (g *Generator) resolveRecordSchema(schema *avro.RecordSchema) string {
 
 	typeName := g.resolveTypeName(schema)
 	if !g.hasTypeDef(typeName) {
-		g.typedefs = append(g.typedefs, newType(typeName, schema.Doc(), fields, schema.String(), schema.Props()))
+		g.typedefs = append(g.typedefs, newType(typeName, schema.Doc(), fields, g.rawSchema(schema), schema.Props()))
 	}
 	return typeName
 }
 
+func (g *Generator) rawSchema(schema *avro.RecordSchema) string {
+	if g.fullSchema {
+		schemaJSON, err := schema.MarshalJSON()
+		if err != nil {
+			panic(fmt.Errorf("failed to marshal raw schema for '%s': %w", schema.FullName(), err))
+		}
+		return string(schemaJSON)
+	}
+	return schema.String()
+}
+
 func (g *Generator) hasTypeDef(name string) bool {
 	for _, def := range g.typedefs {
 		if def.Name != name {


Test patch:
diff --git a/cmd/avrogen/main_test.go b/cmd/avrogen/main_test.go
index 4788be0..c15c3c4 100644
--- a/cmd/avrogen/main_test.go
+++ b/cmd/avrogen/main_test.go
@@ -143,6 +143,29 @@ func TestAvroGen_GeneratesSchemaWithEncoders(t *testing.T) {
 	assert.Equal(t, want, got)
 }
 
+func TestAvroGen_GeneratesSchemaWithFullSchema(t *testing.T) {
+	path, err := os.MkdirTemp("./", "avrogen")
+	require.NoError(t, err)
+	t.Cleanup(func() { _ = os.RemoveAll(path) })
+
+	file := filepath.Join(path, "test.go")
+	args := []string{"avrogen", "-pkg", "testpkg", "-o", file, "-encoders", "-fullschema", "testdata/schema.avsc"}
+	gotCode := realMain(args, io.Discard, io.Discard)
+	require.Equal(t, 0, gotCode)
+
+	got, err := os.ReadFile(file)
+	require.NoError(t, err)
+
+	if *update {
+		err = os.WriteFile("testdata/golden_encoders_fullschema.go", got, 0600)
+		require.NoError(t, err)
+	}
+
+	want, err := os.ReadFile("testdata/golden_encoders_fullschema.go")
+	require.NoError(t, err)
+	assert.Equal(t, want, got)
+}
+
 func TestAvroGen_GeneratesSchemaWithStrictTypes(t *testing.T) {
 	path, err := os.MkdirTemp("./", "avrogen")
 	require.NoError(t, err)
diff --git a/cmd/avrogen/testdata/golden_encoders_fullschema.go b/cmd/avrogen/testdata/golden_encoders_fullschema.go
new file mode 100644
index 0000000..721f0f7
--- /dev/null
+++ b/cmd/avrogen/testdata/golden_encoders_fullschema.go
@@ -0,0 +1,31 @@
+package testpkg
+
+// Code generated by avro/gen. DO NOT EDIT.
+
+import (
+	"github.com/hamba/avro/v2"
+)
+
+// Test is a test struct.
+type Test struct {
+	// SomeString is a string.
+	SomeString string `avro:"someString"`
+	SomeInt    int    `avro:"someInt"`
+}
+
+var schemaTest = avro.MustParse(`{"name":"a.b.test","doc":"Test is a test struct","type":"record","fields":[{"name":"someString","doc":"SomeString is a string","type":"string"},{"name":"someInt","type":"int"}]}`)
+
+// Schema returns the schema for Test.
+func (o *Test) Schema() avro.Schema {
+	return schemaTest
+}
+
+// Unmarshal decodes b into the receiver.
+func (o *Test) Unmarshal(b []byte) error {
+	return avro.Unmarshal(o.Schema(), b, o)
+}
+
+// Marshal encodes the receiver.
+func (o *Test) Marshal() ([]byte, error) {
+	return avro.Marshal(o.Schema(), o)
+}
diff --git a/gen/gen_test.go b/gen/gen_test.go
index 5d7c913..cceede1 100644
--- a/gen/gen_test.go
+++ b/gen/gen_test.go
@@ -261,6 +261,23 @@ func TestStruct_GenFromRecordSchemaWithEncoders(t *testing.T) {
 	assert.Equal(t, string(want), string(file))
 }
 
+func TestStruct_GenFromRecordSchemaWithFullSchema(t *testing.T) {
+	schema, err := os.ReadFile("testdata/golden.avsc")
+	require.NoError(t, err)
+
+	gc := gen.Config{PackageName: "Something", FullSchema: true, Encoders: true}
+	file, _ := generate(t, string(schema), gc)
+
+	if *update {
+		err = os.WriteFile("testdata/golden_encoders_fullschema.go", file, 0600)
+		require.NoError(t, err)
+	}
+
+	want, err := os.ReadFile("testdata/golden_encoders_fullschema.go")
+	require.NoError(t, err)
+	assert.Equal(t, string(want), string(file))
+}
+
 func TestGenerator(t *testing.T) {
 	unionSchema, err := avro.ParseFiles("testdata/uniontype.avsc")
 	require.NoError(t, err)
diff --git a/gen/testdata/golden_encoders_fullschema.go b/gen/testdata/golden_encoders_fullschema.go
new file mode 100644
index 0000000..908c13a
--- /dev/null
+++ b/gen/testdata/golden_encoders_fullschema.go
@@ -0,0 +1,237 @@
+package something
+
+// Code generated by avro/gen. DO NOT EDIT.
+
+import (
+	"math/big"
+	"time"
+
+	"github.com/hamba/avro/v2"
+)
+
+// InnerRecord is a generated struct.
+type InnerRecord struct {
+	InnerJustBytes                   []byte    `avro:"innerJustBytes"`
+	InnerPrimitiveNullableArrayUnion *[]string `avro:"innerPrimitiveNullableArrayUnion"`
+}
+
+var schemaInnerRecord = avro.MustParse(`{"name":"a.c.InnerRecord","type":"record","fields":[{"name":"innerJustBytes","type":"bytes"},{"name":"innerPrimitiveNullableArrayUnion","type":["null",{"type":"array","items":"string"}],"default":null}]}`)
+
+// Schema returns the schema for InnerRecord.
+func (o *InnerRecord) Schema() avro.Schema {
+	return schemaInnerRecord
+}
+
+// Unmarshal decodes b into the receiver.
+func (o *InnerRecord) Unmarshal(b []byte) error {
+	return avro.Unmarshal(o.Schema(), b, o)
+}
+
+// Marshal encodes the receiver.
+func (o *InnerRecord) Marshal() ([]byte, error) {
+	return avro.Marshal(o.Schema(), o)
+}
+
+// RecordInMap is a generated struct.
+type RecordInMap struct {
+	Name string `avro:"name"`
+}
+
+var schemaRecordInMap = avro.MustParse(`{"name":"a.b.RecordInMap","type":"record","fields":[{"name":"name","type":"string"}]}`)
+
+// Schema returns the schema for RecordInMap.
+func (o *RecordInMap) Schema() avro.Schema {
+	return schemaRecordInMap
+}
+
+// Unmarshal decodes b into the receiver.
+func (o *RecordInMap) Unmarshal(b []byte) error {
+	return avro.Unmarshal(o.Schema(), b, o)
+}
+
+// Marshal encodes the receiver.
+func (o *RecordInMap) Marshal() ([]byte, error) {
+	return avro.Marshal(o.Schema(), o)
+}
+
+// RecordInArray is a generated struct.
+type RecordInArray struct {
+	AString string `avro:"aString"`
+}
+
+var schemaRecordInArray = avro.MustParse(`{"name":"a.b.recordInArray","type":"record","fields":[{"name":"aString","type":"string"}]}`)
+
+// Schema returns the schema for RecordInArray.
+func (o *RecordInArray) Schema() avro.Schema {
+	return schemaRecordInArray
+}
+
+// Unmarshal decodes b into the receiver.
+func (o *RecordInArray) Unmarshal(b []byte) error {
+	return avro.Unmarshal(o.Schema(), b, o)
+}
+
+// Marshal encodes the receiver.
+func (o *RecordInArray) Marshal() ([]byte, error) {
+	return avro.Marshal(o.Schema(), o)
+}
+
+// RecordInNullableUnion is a generated struct.
+type RecordInNullableUnion struct {
+	AString string `avro:"aString"`
+}
+
+var schemaRecordInNullableUnion = avro.MustParse(`{"name":"a.b.recordInNullableUnion","type":"record","fields":[{"name":"aString","type":"string"}]}`)
+
+// Schema returns the schema for RecordInNullableUnion.
+func (o *RecordInNullableUnion) Schema() avro.Schema {
+	return schemaRecordInNullableUnion
+}
+
+// Unmarshal decodes b into the receiver.
+func (o *RecordInNullableUnion) Unmarshal(b []byte) error {
+	return avro.Unmarshal(o.Schema(), b, o)
+}
+
+// Marshal encodes the receiver.
+func (o *RecordInNullableUnion) Marshal() ([]byte, error) {
+	return avro.Marshal(o.Schema(), o)
+}
+
+// Record1InNonNullableUnion is a generated struct.
+type Record1InNonNullableUnion struct {
+	AString string `avro:"aString"`
+}
+
+var schemaRecord1InNonNullableUnion = avro.MustParse(`{"name":"a.b.record1InNonNullableUnion","type":"record","fields":[{"name":"aString","type":"string"}]}`)
+
+// Schema returns the schema for Record1InNonNullableUnion.
+func (o *Record1InNonNullableUnion) Schema() avro.Schema {
+	return schemaRecord1InNonNullableUnion
+}
+
+// Unmarshal decodes b into the receiver.
+func (o *Record1InNonNullableUnion) Unmarshal(b []byte) error {
+	return avro.Unmarshal(o.Schema(), b, o)
+}
+
+// Marshal encodes the receiver.
+func (o *Record1InNonNullableUnion) Marshal() ([]byte, error) {
+	return avro.Marshal(o.Schema(), o)
+}
+
+// Record2InNonNullableUnion is a generated struct.
+type Record2InNonNullableUnion struct {
+	AString string `avro:"aString"`
+}
+
+var schemaRecord2InNonNullableUnion = avro.MustParse(`{"name":"a.b.record2InNonNullableUnion","type":"record","fields":[{"name":"aString","type":"string"}]}`)
+
+// Schema returns the schema for Record2InNonNullableUnion.
+func (o *Record2InNonNullableUnion) Schema() avro.Schema {
+	return schemaRecord2InNonNullableUnion
+}
+
+// Unmarshal decodes b into the receiver.
+func (o *Record2InNonNullableUnion) Unmarshal(b []byte) error {
+	return avro.Unmarshal(o.Schema(), b, o)
+}
+
+// Marshal encodes the receiver.
+func (o *Record2InNonNullableUnion) Marshal() ([]byte, error) {
+	return avro.Marshal(o.Schema(), o)
+}
+
+// Record1InNullableUnion is a generated struct.
+type Record1InNullableUnion struct {
+	AString string `avro:"aString"`
+}
+
+var schemaRecord1InNullableUnion = avro.MustParse(`{"name":"a.b.record1InNullableUnion","type":"record","fields":[{"name":"aString","type":"string"}]}`)
+
+// Schema returns the schema for Record1InNullableUnion.
+func (o *Record1InNullableUnion) Schema() avro.Schema {
+	return schemaRecord1InNullableUnion
+}
+
+// Unmarshal decodes b into the receiver.
+func (o *Record1InNullableUnion) Unmarshal(b []byte) error {
+	return avro.Unmarshal(o.Schema(), b, o)
+}
+
+// Marshal encodes the receiver.
+func (o *Record1InNullableUnion) Marshal() ([]byte, error) {
+	return avro.Marshal(o.Schema(), o)
+}
+
+// Record2InNullableUnion is a generated struct.
+type Record2InNullableUnion struct {
+	AString string `avro:"aString"`
+}
+
+var schemaRecord2InNullableUnion = avro.MustParse(`{"name":"a.b.record2InNullableUnion","type":"record","fields":[{"name":"aString","type":"string"}]}`)
+
+// Schema returns the schema for Record2InNullableUnion.
+func (o *Record2InNullableUnion) Schema() avro.Schema {
+	return schemaRecord2InNullableUnion
+}
+
+// Unmarshal decodes b into the receiver.
+func (o *Record2InNullableUnion) Unmarshal(b []byte) error {
+	return avro.Unmarshal(o.Schema(), b, o)
+}
+
+// Marshal encodes the receiver.
+func (o *Record2InNullableUnion) Marshal() ([]byte, error) {
+	return avro.Marshal(o.Schema(), o)
+}
+
+// Test represents a golden record.
+type Test struct {
+	// aString is just a string.
+	AString string `avro:"aString"`
+	// aBoolean is just a boolean.
+	ABoolean                        bool                   `avro:"aBoolean"`
+	AnInt                           int                    `avro:"anInt"`
+	AFloat                          float32                `avro:"aFloat"`
+	ADouble                         float64                `avro:"aDouble"`
+	ALong                           int64                  `avro:"aLong"`
+	JustBytes                       []byte                 `avro:"justBytes"`
+	PrimitiveNullableArrayUnion     *[]string              `avro:"primitiveNullableArrayUnion"`
+	InnerRecord                     InnerRecord            `avro:"innerRecord"`
+	AnEnum                          string                 `avro:"anEnum"`
+	AFixed                          [7]byte                `avro:"aFixed"`
+	ALogicalFixed                   avro.LogicalDuration   `avro:"aLogicalFixed"`
+	AnotherLogicalFixed             avro.LogicalDuration   `avro:"anotherLogicalFixed"`
+	MapOfStrings                    map[string]string      `avro:"mapOfStrings"`
+	MapOfRecords                    map[string]RecordInMap `avro:"mapOfRecords"`
+	ADate                           time.Time              `avro:"aDate"`
+	ADuration                       time.Duration          `avro:"aDuration"`
+	ALongTimeMicros                 time.Duration          `avro:"aLongTimeMicros"`
+	ALongTimestampMillis            time.Time              `avro:"aLongTimestampMillis"`
+	ALongTimestampMicro             time.Time              `avro:"aLongTimestampMicro"`
+	ABytesDecimal                   *big.Rat               `avro:"aBytesDecimal"`
+	ARecordArray                    []RecordInArray        `avro:"aRecordArray"`
+	NullableRecordUnion             *RecordInNullableUnion `avro:"nullableRecordUnion"`
+	NonNullableRecordUnion          any                    `avro:"nonNullableRecordUnion"`
+	NullableRecordUnionWith3Options any                    `avro:"nullableRecordUnionWith3Options"`
+	Ref                             Record2InNullableUnion `avro:"ref"`
+	UUID                            string                 `avro:"uuid"`
+}
+
+var schemaTest = avro.MustParse(`{"name":"a.b.test","doc":"Test represents a golden record","type":"record","fields":[{"name":"aString","doc":"aString is just a string","type":"string"},{"name":"aBoolean","doc":"aBoolean is just a boolean.","type":"boolean"},{"name":"anInt","type":"int"},{"name":"aFloat","type":"float"},{"name":"aDouble","type":"double"},{"name":"aLong","type":"long"},{"name":"justBytes","type":"bytes"},{"name":"primitiveNullableArrayUnion","type":["null",{"type":"array","items":"string"}],"default":null},{"name":"innerRecord","type":{"name":"a.c.InnerRecord","type":"record","fields":[{"name":"innerJustBytes","type":"bytes"},{"name":"innerPrimitiveNullableArrayUnion","type":["null",{"type":"array","items":"string"}],"default":null}]}},{"name":"anEnum","type":{"name":"a.b.Cards","type":"enum","symbols":["SPADES","HEARTS","DIAMONDS","CLUBS"]}},{"name":"aFixed","type":{"name":"a.b.fixedField","type":"fixed","size":7}},{"name":"aLogicalFixed","type":{"name":"a.b.logicalDuration","type":"fixed","size":12,"logicalType":"duration"}},{"name":"anotherLogicalFixed","type":"a.b.logicalDuration"},{"name":"mapOfStrings","type":{"type":"map","values":"string"}},{"name":"mapOfRecords","type":{"type":"map","values":{"name":"a.b.RecordInMap","type":"record","fields":[{"name":"name","type":"string"}]}}},{"name":"aDate","type":{"type":"int","logicalType":"date"}},{"name":"aDuration","type":{"type":"int","logicalType":"time-millis"}},{"name":"aLongTimeMicros","type":{"type":"long","logicalType":"time-micros"}},{"name":"aLongTimestampMillis","type":{"type":"long","logicalType":"timestamp-millis"}},{"name":"aLongTimestampMicro","type":{"type":"long","logicalType":"timestamp-micros"}},{"name":"aBytesDecimal","type":{"type":"bytes","logicalType":"decimal","precision":4,"scale":2}},{"name":"aRecordArray","type":{"type":"array","items":{"name":"a.b.recordInArray","type":"record","fields":[{"name":"aString","type":"string"}]}}},{"name":"nullableRecordUnion","type":["null",{"name":"a.b.recordInNullableUnion","type":"record","fields":[{"name":"aString","type":"string"}]}],"default":null},{"name":"nonNullableRecordUnion","type":[{"name":"a.b.record1InNonNullableUnion","type":"record","fields":[{"name":"aString","type":"string"}]},{"name":"a.b.record2InNonNullableUnion","type":"record","fields":[{"name":"aString","type":"string"}]}]},{"name":"nullableRecordUnionWith3Options","type":["null",{"name":"a.b.record1InNullableUnion","type":"record","fields":[{"name":"aString","type":"string"}]},{"name":"a.b.record2InNullableUnion","type":"record","fields":[{"name":"aString","type":"string"}]}],"default":null},{"name":"ref","type":"a.b.record2InNullableUnion"},{"name":"uuid","type":{"type":"string","logicalType":"uuid"}}]}`)
+
+// Schema returns the schema for Test.
+func (o *Test) Schema() avro.Schema {
+	return schemaTest
+}
+
+// Unmarshal decodes b into the receiver.
+func (o *Test) Unmarshal(b []byte) error {
+	return avro.Unmarshal(o.Schema(), b, o)
+}
+
+// Marshal encodes the receiver.
+func (o *Test) Marshal() ([]byte, error) {
+	return avro.Marshal(o.Schema(), o)
+}