Repository: hamba/avro
Pull request #152 (2022-04-22 13:48:11)

Add support for large scale decimals
I've used your package to work with BigQuery. It has [`BIGNUMERIC`](https://cloud.google.com/bigquery/docs/reference/standard-sql/data-types#decimal_types) data type which is encoded to avro as `bytes.decimal`:

<img width="858" alt="image" src="https://user-images.githubusercontent.com/8282530/164726879-2ca0d88a-0fc2-4cd7-9782-1a4265113256.png">

So package was not working initially with this data type because it has a scale which overflows `int64` exponent.

This PR adds exponent computation using `big` package instead of `math.Exponent` which allows to work with high scale decimals.


Patch:
diff --git a/codec_fixed.go b/codec_fixed.go
index 416fbf0..5f7b62d 100644
--- a/codec_fixed.go
+++ b/codec_fixed.go
@@ -2,7 +2,6 @@ package avro
 
 import (
 	"fmt"
-	"math"
 	"math/big"
 	"reflect"
 	"unsafe"
@@ -89,7 +88,8 @@ func (c *fixedDecimalCodec) Decode(ptr unsafe.Pointer, r *Reader) {
 
 func (c *fixedDecimalCodec) Encode(ptr unsafe.Pointer, w *Writer) {
 	r := *((**big.Rat)(ptr))
-	i := (&big.Int{}).Mul(r.Num(), big.NewInt(int64(math.Pow10(c.scale))))
+	scale := new(big.Int).Exp(big.NewInt(10), big.NewInt(int64(c.scale)), nil)
+	i := (&big.Int{}).Mul(r.Num(), scale)
 	i = i.Div(i, r.Denom())
 
 	var b []byte
diff --git a/codec_native.go b/codec_native.go
index 4e3c2c9..2e69cb7 100644
--- a/codec_native.go
+++ b/codec_native.go
@@ -2,7 +2,6 @@ package avro
 
 import (
 	"fmt"
-	"math"
 	"math/big"
 	"reflect"
 	"time"
@@ -469,16 +468,18 @@ func (c *bytesDecimalCodec) Decode(ptr unsafe.Pointer, r *Reader) {
 }
 
 func ratFromBytes(b []byte, scale int) *big.Rat {
-	i := (&big.Int{}).SetBytes(b)
+	num := (&big.Int{}).SetBytes(b)
 	if len(b) > 0 && b[0]&0x80 > 0 {
-		i.Sub(i, new(big.Int).Lsh(one, uint(len(b))*8))
+		num.Sub(num, new(big.Int).Lsh(one, uint(len(b))*8))
 	}
-	return big.NewRat(i.Int64(), int64(math.Pow10(scale)))
+	denom := new(big.Int).Exp(big.NewInt(10), big.NewInt(int64(scale)), nil)
+	return new(big.Rat).SetFrac(num, denom)
 }
 
 func (c *bytesDecimalCodec) Encode(ptr unsafe.Pointer, w *Writer) {
 	r := (*big.Rat)(ptr)
-	i := (&big.Int{}).Mul(r.Num(), big.NewInt(int64(math.Pow10(c.scale))))
+	scale := new(big.Int).Exp(big.NewInt(10), big.NewInt(int64(c.scale)), nil)
+	i := (&big.Int{}).Mul(r.Num(), scale)
 	i = i.Div(i, r.Denom())
 
 	var b []byte
@@ -514,7 +515,8 @@ func (c *bytesDecimalPtrCodec) Decode(ptr unsafe.Pointer, r *Reader) {
 
 func (c *bytesDecimalPtrCodec) Encode(ptr unsafe.Pointer, w *Writer) {
 	r := *((**big.Rat)(ptr))
-	i := (&big.Int{}).Mul(r.Num(), big.NewInt(int64(math.Pow10(c.scale))))
+	scale := new(big.Int).Exp(big.NewInt(10), big.NewInt(int64(c.scale)), nil)
+	i := (&big.Int{}).Mul(r.Num(), scale)
 	i = i.Div(i, r.Denom())
 
 	var b []byte


Test patch:
diff --git a/decoder_union_test.go b/decoder_union_test.go
index d1421b0..e6c33a1 100644
--- a/decoder_union_test.go
+++ b/decoder_union_test.go
@@ -97,15 +97,29 @@ func TestDecoder_UnionMapWithDuration(t *testing.T) {
 func TestDecoder_UnionMapWithDecimal(t *testing.T) {
 	defer ConfigTeardown()
 
-	data := []byte{0x02, 0x6, 0x00, 0x87, 0x78}
-	schema := `["null", {"type": "bytes", "logicalType": "decimal", "precision": 4, "scale": 2}]`
-	dec, _ := avro.NewDecoder(schema, bytes.NewReader(data))
+	t.Run("low scale", func(t *testing.T) {
+		data := []byte{0x02, 0x6, 0x00, 0x87, 0x78}
+		schema := `["null", {"type": "bytes", "logicalType": "decimal", "precision": 4, "scale": 2}]`
+		dec, _ := avro.NewDecoder(schema, bytes.NewReader(data))
 
-	var got map[string]interface{}
-	err := dec.Decode(&got)
+		var got map[string]interface{}
+		err := dec.Decode(&got)
 
-	assert.NoError(t, err)
-	assert.Equal(t, big.NewRat(1734, 5), got["bytes.decimal"])
+		assert.NoError(t, err)
+		assert.Equal(t, big.NewRat(1734, 5), got["bytes.decimal"])
+	})
+
+	t.Run("high scale", func(t *testing.T) {
+		data := []byte{0x2, 0x22, 0x65, 0xea, 0x55, 0xc, 0x11, 0x8, 0xf7, 0xc3, 0xb8, 0xec, 0x53, 0xff, 0x80, 0x0, 0x0, 0x0, 0x0}
+		schema := `["null", {"type": "bytes", "logicalType": "decimal", "precision": 77, "scale": 38}]`
+		dec, _ := avro.NewDecoder(schema, bytes.NewReader(data))
+
+		var got map[string]interface{}
+		err := dec.Decode(&got)
+
+		assert.NoError(t, err)
+		assert.Equal(t, big.NewRat(1734, 5), got["bytes.decimal"])
+	})
 }
 
 func TestDecoder_UnionMapInvalidSchema(t *testing.T) {
@@ -481,29 +495,56 @@ func TestDecoder_UnionInterfaceWithDuration(t *testing.T) {
 func TestDecoder_UnionInterfaceWithDecimal(t *testing.T) {
 	defer ConfigTeardown()
 
-	data := []byte{0x02, 0x6, 0x00, 0x87, 0x78}
-	schema := `["null", {"type": "bytes", "logicalType": "decimal", "precision": 4, "scale": 2}]`
-	dec, _ := avro.NewDecoder(schema, bytes.NewReader(data))
+	t.Run("low scale", func(t *testing.T) {
+		data := []byte{0x02, 0x6, 0x00, 0x87, 0x78}
+		schema := `["null", {"type": "bytes", "logicalType": "decimal", "precision": 4, "scale": 2}]`
+		dec, _ := avro.NewDecoder(schema, bytes.NewReader(data))
 
-	var got interface{}
-	err := dec.Decode(&got)
+		var got interface{}
+		err := dec.Decode(&got)
 
-	assert.NoError(t, err)
-	assert.Equal(t, big.NewRat(1734, 5), got)
+		assert.NoError(t, err)
+		assert.Equal(t, big.NewRat(1734, 5), got)
+	})
+
+	t.Run("high scale", func(t *testing.T) {
+		data := []byte{0x2, 0x22, 0x65, 0xea, 0x55, 0xc, 0x11, 0x8, 0xf7, 0xc3, 0xb8, 0xec, 0x53, 0xff, 0x80, 0x0, 0x0, 0x0, 0x0}
+		schema := `["null", {"type": "bytes", "logicalType": "decimal", "precision": 77, "scale": 38}]`
+		dec, _ := avro.NewDecoder(schema, bytes.NewReader(data))
+
+		var got interface{}
+		err := dec.Decode(&got)
+
+		assert.NoError(t, err)
+		assert.Equal(t, big.NewRat(1734, 5), got)
+	})
 }
 
 func TestDecoder_UnionInterfaceWithDecimal_Negative(t *testing.T) {
 	defer ConfigTeardown()
 
-	data := []byte{0x02, 0x6, 0xFF, 0x78, 0x88}
-	schema := `["null", {"type": "bytes", "logicalType": "decimal", "precision": 4, "scale": 2}]`
-	dec, _ := avro.NewDecoder(schema, bytes.NewReader(data))
+	t.Run("low scale", func(t *testing.T) {
+		data := []byte{0x02, 0x6, 0xFF, 0x78, 0x88}
+		schema := `["null", {"type": "bytes", "logicalType": "decimal", "precision": 4, "scale": 2}]`
+		dec, _ := avro.NewDecoder(schema, bytes.NewReader(data))
 
-	var got interface{}
-	err := dec.Decode(&got)
+		var got interface{}
+		err := dec.Decode(&got)
 
-	assert.NoError(t, err)
-	assert.Equal(t, big.NewRat(-1734, 5), got)
+		assert.NoError(t, err)
+		assert.Equal(t, big.NewRat(-1734, 5), got)
+	})
+	t.Run("high scale", func(t *testing.T) {
+		data := []byte{0x2, 0x22, 0x9a, 0x15, 0xaa, 0xf3, 0xee, 0xf7, 0x8, 0x3c, 0x47, 0x13, 0xac, 0x0, 0x80, 0x0, 0x0, 0x0, 0x0}
+		schema := `["null", {"type": "bytes", "logicalType": "decimal", "precision": 77, "scale": 38}]`
+		dec, _ := avro.NewDecoder(schema, bytes.NewReader(data))
+
+		var got interface{}
+		err := dec.Decode(&got)
+
+		assert.NoError(t, err)
+		assert.Equal(t, big.NewRat(-1734, 5), got)
+	})
 }
 
 func TestDecoder_UnionInterfaceUnresolvableTypeWithError(t *testing.T) {
diff --git a/encoder_union_test.go b/encoder_union_test.go
index 8b89f39..1ddd558 100644
--- a/encoder_union_test.go
+++ b/encoder_union_test.go
@@ -124,18 +124,35 @@ func TestEncoder_UnionMapWithDuration(t *testing.T) {
 func TestEncoder_UnionMapWithDecimal(t *testing.T) {
 	defer ConfigTeardown()
 
-	schema := `["null", {"type": "bytes", "logicalType": "decimal", "precision": 4, "scale": 2}]`
-	buf := bytes.NewBuffer([]byte{})
-	enc, err := avro.NewEncoder(schema, buf)
-	assert.NoError(t, err)
-
-	m := map[string]interface{}{
-		"bytes.decimal": big.NewRat(1734, 5),
-	}
-	err = enc.Encode(m)
-
-	assert.NoError(t, err)
-	assert.Equal(t, []byte{0x02, 0x6, 0x00, 0x87, 0x78}, buf.Bytes())
+	t.Run("low scale", func(t *testing.T) {
+		schema := `["null", {"type": "bytes", "logicalType": "decimal", "precision": 4, "scale": 2}]`
+		buf := bytes.NewBuffer([]byte{})
+		enc, err := avro.NewEncoder(schema, buf)
+		assert.NoError(t, err)
+
+		m := map[string]interface{}{
+			"bytes.decimal": big.NewRat(1734, 5),
+		}
+		err = enc.Encode(m)
+
+		assert.NoError(t, err)
+		assert.Equal(t, []byte{0x02, 0x6, 0x00, 0x87, 0x78}, buf.Bytes())
+	})
+
+	t.Run("high scale", func(t *testing.T) {
+		schema := `["null", {"type": "bytes", "logicalType": "decimal", "precision": 77, "scale": 38}]`
+		buf := bytes.NewBuffer([]byte{})
+		enc, err := avro.NewEncoder(schema, buf)
+		assert.NoError(t, err)
+
+		m := map[string]interface{}{
+			"bytes.decimal": big.NewRat(1734, 5),
+		}
+		err = enc.Encode(m)
+
+		assert.NoError(t, err)
+		assert.Equal(t, []byte{0x2, 0x22, 0x65, 0xea, 0x55, 0xc, 0x11, 0x8, 0xf7, 0xc3, 0xb8, 0xec, 0x53, 0xff, 0x80, 0x0, 0x0, 0x0, 0x0}, buf.Bytes())
+	})
 }
 
 func TestEncoder_UnionMapInvalidType(t *testing.T) {
@@ -399,16 +416,32 @@ func TestEncoder_UnionInterfaceWithDuration(t *testing.T) {
 func TestEncoder_UnionInterfaceWithDecimal(t *testing.T) {
 	defer ConfigTeardown()
 
-	schema := `["null", {"type": "bytes", "logicalType": "decimal", "precision": 4, "scale": 2}]`
-	buf := bytes.NewBuffer([]byte{})
-	enc, err := avro.NewEncoder(schema, buf)
-	assert.NoError(t, err)
+	t.Run("low scale", func(t *testing.T) {
+		schema := `["null", {"type": "bytes", "logicalType": "decimal", "precision": 4, "scale": 2}]`
+		buf := bytes.NewBuffer([]byte{})
+		enc, err := avro.NewEncoder(schema, buf)
+		assert.NoError(t, err)
 
-	var val interface{} = big.NewRat(1734, 5)
-	err = enc.Encode(val)
+		var val interface{} = big.NewRat(1734, 5)
+		err = enc.Encode(val)
+
+		assert.NoError(t, err)
+		assert.Equal(t, []byte{0x02, 0x6, 0x00, 0x87, 0x78}, buf.Bytes())
+	})
+
+	t.Run("high scale", func(t *testing.T) {
+		schema := `["null", {"type": "bytes", "logicalType": "decimal", "precision": 77, "scale": 38}]`
+		buf := bytes.NewBuffer([]byte{})
+		enc, err := avro.NewEncoder(schema, buf)
+		assert.NoError(t, err)
+
+		var val interface{} = big.NewRat(1734, 5)
+		err = enc.Encode(val)
+
+		assert.NoError(t, err)
+		assert.Equal(t, []byte{0x2, 0x22, 0x65, 0xea, 0x55, 0xc, 0x11, 0x8, 0xf7, 0xc3, 0xb8, 0xec, 0x53, 0xff, 0x80, 0x0, 0x0, 0x0, 0x0}, buf.Bytes())
+	})
 
-	assert.NoError(t, err)
-	assert.Equal(t, []byte{0x02, 0x6, 0x00, 0x87, 0x78}, buf.Bytes())
 }
 
 func TestEncoder_UnionInterfaceUnregisteredType(t *testing.T) {