Repository: hamba/avro
Pull request #2 (2019-03-05 06:57:16)

Docs


Patch:
diff --git a/README.md b/README.md
index 2ac1f39..2d652be 100644
--- a/README.md
+++ b/README.md
@@ -58,6 +58,8 @@ fmt.Println(out)
 // Outputs: {27 foo}
 ```
 
+More examples in the [godoc](https://godoc.org/github.com/hamba/avro).
+
 #### Types Conversions
 
 | Avro     | Go Struct                          | Go Interface              |
@@ -116,4 +118,5 @@ Always benchmark with your own workload. The result depends heavily on the data
     * Refactor parsing to be cleaner
     * Better schema validation
     * Aliases
+* Avro container file codecs
 * Avro Textual Form?
diff --git a/avro.go b/avro.go
index 64114b3..8c4d82a 100644
--- a/avro.go
+++ b/avro.go
@@ -1,7 +1,7 @@
 package avro
 
 // UnionType is the interface implemented by types that can
-// resolve schema types/names to and from objects
+// resolve schema types/names to and from objects.
 type UnionType interface {
 	Value() *interface{}
 	SetType(string) error
diff --git a/container/container.go b/container/container.go
index 279e53a..b6edbc6 100644
--- a/container/container.go
+++ b/container/container.go
@@ -1,3 +1,9 @@
+/*
+Package container implements encoding and decoding of Avro Object Container Files as defined by the Avro specification.
+
+See the Avro specification for an understanding of Avro: http://avro.apache.org/docs/current/
+
+ */
 package container
 
 import (
@@ -79,15 +85,19 @@ func NewDecoder(r io.Reader) (*Decoder, error) {
 	}, nil
 }
 
+func (d *Decoder) HasNext() bool {
+	if d.count <= 0 {
+		count, _ := d.readBlock() // err handled in Error function
+		d.count = count
+	}
+
+	return d.count > 0
+}
+
 // Decode reads the next Avro encoded value from its input and stores it in the value pointed to by v.
 func (d *Decoder) Decode(v interface{}) error {
 	if d.count <= 0 {
-		count, err := d.readBlock()
-		if err != nil {
-			return err
-		}
-
-		d.count = count
+		return errors.New("decoder: no data found, call HasNext first")
 	}
 
 	d.count--
@@ -99,6 +109,15 @@ func (d *Decoder) Decode(v interface{}) error {
 	return err
 }
 
+// Error returns the last reader error.
+func (d *Decoder) Error() error {
+	if d.reader.Error == io.EOF {
+		return nil
+	}
+
+	return d.reader.Error
+}
+
 func (d *Decoder) readBlock() (int64, error) {
 	count := d.reader.ReadLong()
 	size := d.reader.ReadLong()
@@ -109,7 +128,7 @@ func (d *Decoder) readBlock() (int64, error) {
 
 	var sync [16]byte
 	d.reader.Read(sync[:])
-	if d.sync != sync {
+	if d.sync != sync && d.reader.Error != io.EOF {
 		return count, errors.New("file: invalid block")
 	}
 
diff --git a/doc.go b/doc.go
index 616ad92..7f1b003 100644
--- a/doc.go
+++ b/doc.go
@@ -1,44 +1,7 @@
 /*
 Package avro implements encoding and decoding of Avro as defined by the Avro specification.
 
-See the Avro specification for an understanding of Avro: https://avro.apache.org/docs/1.8.1/spec.html
+See the Avro specification for an understanding of Avro: http://avro.apache.org/docs/current/
 
-Usage Example:
-	type TestRecord struct {
-		A int64  `avro:"a"`
-		B string `avro:"b"`
-	}
-
-	schema, err := avro.Parse(`{
-		"type": "record",
-		"name": "test",
-		"namespace": "org.hamba.avro",
-		"fields" : [
-			{"name": "a", "type": "long"},
-			{"name": "b", "type": "string"}
-		]
-	}`)
-	if err != nil {
-		log.Fatal(err)
-	}
-
-	in := TestRecord{A: 27, B: "foo"}
-
-	data, err := avro.Marshal(schema, in)
-	if err != nil {
-		log.Fatal(err)
-	}
-
-	fmt.Println(data)
-	// Outputs: [54 6 102 111 111]
-
-	out := TestRecord{}
-	err = avro.Unmarshal(schema, data, &out)
-	if err != nil {
-		log.Fatal(err)
-	}
-
-	fmt.Println(out)
-	// Outputs: {27 foo}
 */
 package avro
diff --git a/schema.go b/schema.go
index c1daf8b..2a37486 100644
--- a/schema.go
+++ b/schema.go
@@ -102,17 +102,17 @@ func NewPrimitiveSchema(t Type) *PrimitiveSchema {
 	}
 }
 
-// Type implements the Schema interface.
+// Type returns the type of the schema.
 func (s *PrimitiveSchema) Type() Type {
 	return s.typ
 }
 
-// String implements the Schema interface.
+// String returns the canonical form of the schema.
 func (s *PrimitiveSchema) String() string {
 	return `"` + string(s.typ) + `"`
 }
 
-// Fingerprint implements the Schema interface.
+// Fingerprint returns the SHA256 fingerprint of the schema.
 func (s *PrimitiveSchema) Fingerprint() [32]byte {
 	if s.fingerprint != emptyFgpt {
 		return s.fingerprint
@@ -131,7 +131,7 @@ type RecordSchema struct {
 	fingerprint [32]byte
 }
 
-// Type implements the Schema interface.
+// Type returns the type of the schema.
 func (s *RecordSchema) Type() Type {
 	return Record
 }
@@ -146,7 +146,7 @@ func (s *RecordSchema) Fields() []*Field {
 	return s.fields
 }
 
-// String implements the Schema interface.
+// String returns the canonical form of the schema.
 func (s *RecordSchema) String() string {
 	if s.canonical != "" {
 		return s.canonical
@@ -164,7 +164,7 @@ func (s *RecordSchema) String() string {
 	return s.canonical
 }
 
-// Fingerprint implements the Schema interface.
+// Fingerprint returns the SHA256 fingerprint of the schema.
 func (s *RecordSchema) Fingerprint() [32]byte {
 	if s.fingerprint != emptyFgpt {
 		return s.fingerprint
@@ -212,7 +212,7 @@ type EnumSchema struct {
 	fingerprint [32]byte
 }
 
-// Type implements the Schema interface.
+// Type returns the type of the schema.
 func (s *EnumSchema) Type() Type {
 	return Enum
 }
@@ -227,7 +227,7 @@ func (s *EnumSchema) Symbols() []string {
 	return s.symbols
 }
 
-// String implements the Schema interface.
+// String returns the canonical form of the schema.
 func (s *EnumSchema) String() string {
 	if s.canonical != "" {
 		return s.canonical
@@ -245,7 +245,7 @@ func (s *EnumSchema) String() string {
 	return s.canonical
 }
 
-// Fingerprint implements the Schema interface.
+// Fingerprint returns the SHA256 fingerprint of the schema.
 func (s *EnumSchema) Fingerprint() [32]byte {
 	if s.fingerprint != emptyFgpt {
 		return s.fingerprint
@@ -262,7 +262,7 @@ type ArraySchema struct {
 	fingerprint [32]byte
 }
 
-// Type implements the Schema interface.
+// Type returns the type of the schema.
 func (s *ArraySchema) Type() Type {
 	return Array
 }
@@ -272,12 +272,12 @@ func (s *ArraySchema) Items() Schema {
 	return s.items
 }
 
-// String implements the Schema interface.
+// String returns the canonical form of the schema.
 func (s *ArraySchema) String() string {
 	return `{"type":"array","items":` + s.items.String() + `}`
 }
 
-// Fingerprint implements the Schema interface.
+// Fingerprint returns the SHA256 fingerprint of the schema.
 func (s *ArraySchema) Fingerprint() [32]byte {
 	if s.fingerprint != emptyFgpt {
 		return s.fingerprint
@@ -294,7 +294,7 @@ type MapSchema struct {
 	fingerprint [32]byte
 }
 
-// Type implements the Schema interface.
+// Type returns the type of the schema.
 func (s *MapSchema) Type() Type {
 	return Map
 }
@@ -304,7 +304,7 @@ func (s *MapSchema) Values() Schema {
 	return s.values
 }
 
-// Fingerprint implements the Schema interface.
+// Fingerprint returns the SHA256 fingerprint of the schema.
 func (s *MapSchema) Fingerprint() [32]byte {
 	if s.fingerprint != emptyFgpt {
 		return s.fingerprint
@@ -314,7 +314,7 @@ func (s *MapSchema) Fingerprint() [32]byte {
 	return s.fingerprint
 }
 
-// String implements the Schema interface.
+// String returns the canonical form of the schema.
 func (s *MapSchema) String() string {
 	return `{"type":"map","values":` + s.values.String() + `}`
 }
@@ -327,7 +327,7 @@ type UnionSchema struct {
 	fingerprint [32]byte
 }
 
-// Type implements the Schema interface.
+// Type returns the type of the schema.
 func (s *UnionSchema) Type() Type {
 	return Union
 }
@@ -346,7 +346,7 @@ func (s *UnionSchema) Nullable() bool {
 	return true
 }
 
-// String implements the Schema interface.
+// String returns the canonical form of the schema.
 func (s *UnionSchema) String() string {
 	if s.canonical != "" {
 		return s.canonical
@@ -364,7 +364,7 @@ func (s *UnionSchema) String() string {
 	return s.canonical
 }
 
-// Fingerprint implements the Schema interface.
+// Fingerprint returns the SHA256 fingerprint of the schema.
 func (s *UnionSchema) Fingerprint() [32]byte {
 	if s.fingerprint != emptyFgpt {
 		return s.fingerprint
@@ -383,7 +383,7 @@ type FixedSchema struct {
 	fingerprint [32]byte
 }
 
-// Type implements the Schema interface.
+// Type returns the type of the schema.
 func (s *FixedSchema) Type() Type {
 	return Fixed
 }
@@ -398,7 +398,7 @@ func (s *FixedSchema) Size() int {
 	return s.size
 }
 
-// String implements the Schema interface.
+// String returns the canonical form of the schema.
 func (s *FixedSchema) String() string {
 	if s.canonical != "" {
 		return s.canonical
@@ -409,7 +409,7 @@ func (s *FixedSchema) String() string {
 	return s.canonical
 }
 
-// Fingerprint implements the Schema interface.
+// Fingerprint returns the SHA256 fingerprint of the schema.
 func (s *FixedSchema) Fingerprint() [32]byte {
 	if s.fingerprint != emptyFgpt {
 		return s.fingerprint
@@ -422,17 +422,17 @@ func (s *FixedSchema) Fingerprint() [32]byte {
 // NullSchema is an Avro null type schema.
 type NullSchema struct{}
 
-// Type implements the Schema interface.
+// Type returns the type of the schema.
 func (s *NullSchema) Type() Type {
 	return Null
 }
 
-// String implements the Schema interface.
+// String returns the canonical form of the schema.
 func (s *NullSchema) String() string {
 	return `"null"`
 }
 
-// Fingerprint implements the Schema interface.
+// Fingerprint returns the SHA256 fingerprint of the schema.
 func (s *NullSchema) Fingerprint() [32]byte {
 	return [32]uint8{0xf0, 0x72, 0xcb, 0xec, 0x3b, 0xf8, 0x84, 0x18, 0x71, 0xd4, 0x28, 0x42, 0x30, 0xc5, 0xe9, 0x83, 0xdc, 0x21, 0x1a, 0x56, 0x83, 0x7a, 0xed, 0x86, 0x24, 0x87, 0x14, 0x8f, 0x94, 0x7d, 0x1a, 0x1f}
 }
@@ -442,7 +442,7 @@ type RefSchema struct {
 	actual NamedSchema
 }
 
-// Type implements the Schema interface.
+// Type returns the type of the schema.
 func (s *RefSchema) Type() Type {
 	return Ref
 }
@@ -452,12 +452,12 @@ func (s *RefSchema) Schema() Schema {
 	return s.actual
 }
 
-// String implements the Schema interface.
+// String returns the canonical form of the schema.
 func (s *RefSchema) String() string {
 	return `"` + s.actual.Name() + `"`
 }
 
-// Fingerprint implements the Schema interface.
+// Fingerprint returns the SHA256 fingerprint of the schema.
 func (s *RefSchema) Fingerprint() [32]byte {
 	return s.actual.Fingerprint()
 }


Test patch:
diff --git a/container/container_test.go b/container/container_test.go
index ce36430..005408e 100644
--- a/container/container_test.go
+++ b/container/container_test.go
@@ -92,11 +92,18 @@ func TestDecoder(t *testing.T) {
 		return
 	}
 
-	var got FullRecord
-	err = dec.Decode(&got)
+	var count int
+	for dec.HasNext() {
+		count++
+		var got FullRecord
+		err = dec.Decode(&got)
 
-	assert.NoError(t, err)
-	assert.Equal(t, want, got)
+		assert.NoError(t, err)
+		assert.Equal(t, want, got)
+	}
+
+	assert.NoError(t, dec.Error())
+	assert.Equal(t, 1, count)
 }
 
 func TestEncoder(t *testing.T) {
diff --git a/container/example_test.go b/container/example_test.go
new file mode 100644
index 0000000..7311a82
--- /dev/null
+++ b/container/example_test.go
@@ -0,0 +1,75 @@
+package container_test
+
+import (
+	"log"
+	"os"
+
+	"github.com/hamba/avro/container"
+)
+
+func ExampleNewDecoder() {
+	type SimpleRecord struct {
+		A int64  `avro:"a"`
+		B string `avro:"b"`
+	}
+
+	f, err := os.Open("/your/avro/file.avro")
+	if err != nil {
+		log.Fatal(err)
+	}
+	defer f.Close()
+
+	dec, err := container.NewDecoder(f)
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	for dec.HasNext() {
+		var record SimpleRecord
+		err = dec.Decode(&record)
+		if err != nil {
+			log.Fatal(err)
+		}
+
+		// Do something with the data
+	}
+
+	if dec.Error() != nil {
+		log.Fatal(err)
+	}
+}
+
+func ExampleNewEncoder() {
+	schema := `{
+	    "type": "record",
+	    "name": "simple",
+	    "namespace": "org.hamba.avro",
+	    "fields" : [
+	        {"name": "a", "type": "long"},
+	        {"name": "b", "type": "string"}
+	    ]
+	}`
+
+	type SimpleRecord struct {
+		A int64  `avro:"a"`
+		B string `avro:"b"`
+	}
+
+	f, err := os.Open("/your/avro/file.avro")
+	if err != nil {
+		log.Fatal(err)
+	}
+	defer f.Close()
+
+	enc, err := container.NewEncoder(schema, f)
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	var record SimpleRecord
+	err = enc.Encode(record)
+
+	if err := enc.Close(); err != nil {
+		log.Fatal(err)
+	}
+}
diff --git a/example_simple_test.go b/example_simple_test.go
new file mode 100644
index 0000000..eb69036
--- /dev/null
+++ b/example_simple_test.go
@@ -0,0 +1,49 @@
+package avro_test
+
+import (
+	"fmt"
+	"log"
+
+	"github.com/hamba/avro"
+)
+
+var Schema = `{
+		"type": "record",
+		"name": "simple",
+		"namespace": "org.hamba.avro",
+		"fields" : [
+			{"name": "a", "type": "long"},
+			{"name": "b", "type": "string"}
+		]
+	}`
+
+type SimpleRecord struct {
+	A int64  `avro:"a"`
+	B string `avro:"b"`
+}
+
+func Example_usage() {
+	schema, err := avro.Parse(Schema)
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	in := SimpleRecord{A: 27, B: "foo"}
+
+	data, err := avro.Marshal(schema, in)
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	fmt.Printf("%+v\n", data)
+	// Outputs: [54 6 102 111 111]
+
+	out := SimpleRecord{}
+	err = avro.Unmarshal(schema, data, &out)
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	fmt.Printf("%+v\n", out)
+	// Outputs: {A:27 B:foo}
+}
diff --git a/example_test.go b/example_test.go
index 2fb4a2f..5edf5fc 100644
--- a/example_test.go
+++ b/example_test.go
@@ -3,11 +3,30 @@ package avro_test
 import (
 	"bytes"
 	"fmt"
+	"log"
 	"os"
 
 	"github.com/hamba/avro"
 )
 
+func ExampleParse() {
+	schema, err := avro.Parse(`{
+	    "type": "record",
+	    "name": "simple",
+	    "namespace": "org.hamba.avro",
+	    "fields" : [
+	        {"name": "a", "type": "long"},
+	        {"name": "b", "type": "string"}
+	    ]
+	}`)
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	fmt.Println(schema.Type())
+	// Outputs: record
+}
+
 func ExampleNewDecoder() {
 	schema := `{
 	    "type": "record",
diff --git a/example_uniontype_test.go b/example_uniontype_test.go
new file mode 100644
index 0000000..829f349
--- /dev/null
+++ b/example_uniontype_test.go
@@ -0,0 +1,66 @@
+package avro_test
+
+import (
+	"fmt"
+	"os"
+
+	"github.com/hamba/avro"
+)
+
+type SimpleUnionType struct {
+	Val interface{}
+}
+
+func (u *SimpleUnionType) Value() *interface{} {
+	return &u.Val
+}
+
+func (u *SimpleUnionType) SetType(typ string) error {
+	switch typ {
+	case string(avro.String):
+		u.Val = ""
+
+	case string(avro.Int):
+		u.Val = int(0)
+
+	default:
+		return fmt.Errorf("unknown type %s", typ)
+	}
+
+	return nil
+}
+
+func (u *SimpleUnionType) GetType() (string, error) {
+	switch u.Val.(type) {
+	case string:
+		return string(avro.String), nil
+
+	case int:
+		return string(avro.Int), nil
+	}
+
+	return "", fmt.Errorf("unknown type %T", u.Val)
+}
+
+type UnionRecord struct {
+	Items map[string]*SimpleUnionType `avro:"items"`
+}
+
+func ExampleUnionType() {
+	schema := avro.MustParse(`{
+	    "type": "record",
+	    "name": "union",
+	    "namespace": "org.hamba.avro",
+	    "fields" : [
+	        {"name": "items", "type": {"type": "map", "values": ["string", "int"]}}
+	    ]
+	}`)
+
+	record := UnionRecord{}
+	b, err := avro.Marshal(schema, record)
+	if err != nil {
+		fmt.Println("error:", err)
+	}
+
+	os.Stdout.Write(b)
+}