Repository: hamba/avro
Pull request #210 (2022-11-25 18:01:47)

Added support for unsigned integers
- This supports adds encoding/decoding `uint8` and `uint16` as an Avro `int`, and `uint32` as an Avro `long`.

nrwiersma: Given that the spec does not account for unsigned integers, have you considered the affect of negative numbers on decode and large numbers on encode? 
stampy88: I was treating it similarly to how you were treating `int8` and `int16` in that if you have a 32 bit integer in the Avro space, and actually use all of the bytes, you are going to lose information when converting to an `int8`/`int16`. Could update the `readme` to warn the user of the potential consequences of using Go types that are smaller than the Avro type if they are not careful.
stampy88: I purposefully chose to read/store `uint8`/`uint16` in an `int`, and `uint32` in a `long`. `uint64` can't fit in a `long`, so that isn't supported. 
nrwiersma: Fair. Encoding needs to be supported, and a note on their support in the README should be added.

Patch:
diff --git a/README.md b/README.md
index 4815e63..560d54f 100644
--- a/README.md
+++ b/README.md
@@ -68,29 +68,34 @@ More examples in the [godoc](https://godoc.org/github.com/hamba/avro).
 
 #### Types Conversions
 
-| Avro                    | Go Struct                          | Go Interface              |
-| ----------------------- | ---------------------------------- | ------------------------- |
-| `null`                  | `nil`                              | `nil`                     |
-| `boolean`               | `bool`                             | `bool`                    |
-| `bytes`                 | `[]byte`                           | `[]byte`                  |
-| `float`                 | `float32`                          | `float32`                 |
-| `double`                | `float64`                          | `float64`                 |
-| `long`                  | `int64`                            | `int64`                   |
-| `int`                   | `int`, `int32`, `int16`, `int8`    | `int`                     |
-| `string`                | `string`                           | `string`                  |
-| `array`                 | `[]T`                              | `[]interface{}`           |
-| `enum`                  | `string`                           | `string`                  |
-| `fixed`                 | `[n]byte`                          | `[]byte`                  |
-| `map`                   | `map[string]T{}`                   | `map[string]interface{}`  |
-| `record`                | `struct`                           | `map[string]interface{}`  |
-| `union`                 | *see below*                        | *see below*               |
-| `int.date`              | `time.Time`                        | `time.Time`               |
-| `int.time-millis`       | `time.Duration`                    | `time.Duration`           |
-| `long.time-micros`      | `time.Duration`                    | `time.Duration`           |
-| `long.timestamp-millis` | `time.Time`                        | `time.Time`               |
-| `long.timestamp-micros` | `time.Time`                        | `time.Time`               |
-| `bytes.decimal`         | `*big.Rat`                         | `*big.Rat`                |
-| `fixed.decimal`         | `*big.Rat`                         | `*big.Rat`                |
+| Avro                    | Go Struct                                              | Go Interface             |
+|-------------------------|--------------------------------------------------------|--------------------------|
+| `null`                  | `nil`                                                  | `nil`                    |
+| `boolean`               | `bool`                                                 | `bool`                   |
+| `bytes`                 | `[]byte`                                               | `[]byte`                 |
+| `float`                 | `float32`                                              | `float32`                |
+| `double`                | `float64`                                              | `float64`                |
+| `long`                  | `int64`, `uint32`\*                                    | `int64`, `uint32`        |
+| `int`                   | `int`, `int32`, `int16`, `int8`, `uint8`\*, `uint16`\* | `int`, `uint8`, `uint16` |
+| `string`                | `string`                                               | `string`                 |
+| `array`                 | `[]T`                                                  | `[]interface{}`          |
+| `enum`                  | `string`                                               | `string`                 |
+| `fixed`                 | `[n]byte`                                              | `[]byte`                 |
+| `map`                   | `map[string]T{}`                                       | `map[string]interface{}` |
+| `record`                | `struct`                                               | `map[string]interface{}` |
+| `union`                 | *see below*                                            | *see below*              |
+| `int.date`              | `time.Time`                                            | `time.Time`              |
+| `int.time-millis`       | `time.Duration`                                        | `time.Duration`          |
+| `long.time-micros`      | `time.Duration`                                        | `time.Duration`          |
+| `long.timestamp-millis` | `time.Time`                                            | `time.Time`              |
+| `long.timestamp-micros` | `time.Time`                                            | `time.Time`              |
+| `bytes.decimal`         | `*big.Rat`                                             | `*big.Rat`               |
+| `fixed.decimal`         | `*big.Rat`                                             | `*big.Rat`               |
+
+\* Please note that when the Go type is an unsigned integer care must be taken to ensure that information is not lost 
+when converting between the Avro type and Go type. For example, storing a *negative* number in Avro of `int = -100`
+would be interpreted as `uint16 = 65,436` in Go. Another example would be storing numbers in Avro `int = 256` that 
+are larger than the Go type `uint8 = 0`. 
 
 ##### Unions
 
diff --git a/codec_native.go b/codec_native.go
index 2e69cb7..94d0077 100644
--- a/codec_native.go
+++ b/codec_native.go
@@ -30,18 +30,36 @@ func createDecoderOfNative(schema Schema, typ reflect2.Type) ValDecoder {
 		}
 		return &int8Codec{}
 
+	case reflect.Uint8:
+		if schema.Type() != Int {
+			break
+		}
+		return &uint8Codec{}
+
 	case reflect.Int16:
 		if schema.Type() != Int {
 			break
 		}
 		return &int16Codec{}
 
+	case reflect.Uint16:
+		if schema.Type() != Int {
+			break
+		}
+		return &uint16Codec{}
+
 	case reflect.Int32:
 		if schema.Type() != Int {
 			break
 		}
 		return &int32Codec{}
 
+	case reflect.Uint32:
+		if schema.Type() != Long {
+			break
+		}
+		return &uint32Codec{}
+
 	case reflect.Int64:
 		st := schema.Type()
 		lt := getLogicalType(schema)
@@ -144,12 +162,24 @@ func createEncoderOfNative(schema Schema, typ reflect2.Type) ValEncoder {
 		}
 		return &int8Codec{}
 
+	case reflect.Uint8:
+		if schema.Type() != Int {
+			break
+		}
+		return &uint8Codec{}
+
 	case reflect.Int16:
 		if schema.Type() != Int {
 			break
 		}
 		return &int16Codec{}
 
+	case reflect.Uint16:
+		if schema.Type() != Int {
+			break
+		}
+		return &uint16Codec{}
+
 	case reflect.Int32:
 		switch schema.Type() {
 		case Long:
@@ -159,6 +189,12 @@ func createEncoderOfNative(schema Schema, typ reflect2.Type) ValEncoder {
 			return &int32Codec{}
 		}
 
+	case reflect.Uint32:
+		if schema.Type() != Long {
+			break
+		}
+		return &uint32Codec{}
+
 	case reflect.Int64:
 		st := schema.Type()
 		lt := getLogicalType(schema)
@@ -301,6 +337,16 @@ func (*int8Codec) Encode(ptr unsafe.Pointer, w *Writer) {
 	w.WriteInt(int32(*((*int8)(ptr))))
 }
 
+type uint8Codec struct{}
+
+func (*uint8Codec) Decode(ptr unsafe.Pointer, r *Reader) {
+	*((*uint8)(ptr)) = uint8(r.ReadInt())
+}
+
+func (*uint8Codec) Encode(ptr unsafe.Pointer, w *Writer) {
+	w.WriteInt(int32(*((*uint8)(ptr))))
+}
+
 type int16Codec struct{}
 
 func (*int16Codec) Decode(ptr unsafe.Pointer, r *Reader) {
@@ -311,6 +357,16 @@ func (*int16Codec) Encode(ptr unsafe.Pointer, w *Writer) {
 	w.WriteInt(int32(*((*int16)(ptr))))
 }
 
+type uint16Codec struct{}
+
+func (*uint16Codec) Decode(ptr unsafe.Pointer, r *Reader) {
+	*((*uint16)(ptr)) = uint16(r.ReadInt())
+}
+
+func (*uint16Codec) Encode(ptr unsafe.Pointer, w *Writer) {
+	w.WriteInt(int32(*((*uint16)(ptr))))
+}
+
 type int32Codec struct{}
 
 func (*int32Codec) Decode(ptr unsafe.Pointer, r *Reader) {
@@ -321,6 +377,16 @@ func (*int32Codec) Encode(ptr unsafe.Pointer, w *Writer) {
 	w.WriteInt(*((*int32)(ptr)))
 }
 
+type uint32Codec struct{}
+
+func (*uint32Codec) Decode(ptr unsafe.Pointer, r *Reader) {
+	*((*uint32)(ptr)) = uint32(r.ReadLong())
+}
+
+func (*uint32Codec) Encode(ptr unsafe.Pointer, w *Writer) {
+	w.WriteLong(int64(*((*uint32)(ptr))))
+}
+
 type int32LongCodec struct{}
 
 func (*int32LongCodec) Encode(ptr unsafe.Pointer, w *Writer) {


Test patch:
diff --git a/decoder_native_test.go b/decoder_native_test.go
index 1eae8c5..23e310e 100644
--- a/decoder_native_test.go
+++ b/decoder_native_test.go
@@ -112,6 +112,35 @@ func TestDecoder_Int8InvalidSchema(t *testing.T) {
 	assert.Error(t, err)
 }
 
+func TestDecoder_Uint8(t *testing.T) {
+	defer ConfigTeardown()
+
+	data := []byte{0x36}
+	schema := "int"
+	dec, err := avro.NewDecoder(schema, bytes.NewReader(data))
+	require.NoError(t, err)
+
+	var i uint8
+	err = dec.Decode(&i)
+
+	require.NoError(t, err)
+	assert.Equal(t, uint8(27), i)
+}
+
+func TestDecoder_Uint8InvalidSchema(t *testing.T) {
+	defer ConfigTeardown()
+
+	data := []byte{0x36}
+	schema := "string"
+	dec, err := avro.NewDecoder(schema, bytes.NewReader(data))
+	require.NoError(t, err)
+
+	var i uint8
+	err = dec.Decode(&i)
+
+	assert.Error(t, err)
+}
+
 func TestDecoder_Int16(t *testing.T) {
 	defer ConfigTeardown()
 
@@ -141,6 +170,35 @@ func TestDecoder_Int16InvalidSchema(t *testing.T) {
 	assert.Error(t, err)
 }
 
+func TestDecoder_Uint16(t *testing.T) {
+	defer ConfigTeardown()
+
+	data := []byte{0x36}
+	schema := "int"
+	dec, err := avro.NewDecoder(schema, bytes.NewReader(data))
+	require.NoError(t, err)
+
+	var i uint16
+	err = dec.Decode(&i)
+
+	require.NoError(t, err)
+	assert.Equal(t, uint16(27), i)
+}
+
+func TestDecoder_Uint16InvalidSchema(t *testing.T) {
+	defer ConfigTeardown()
+
+	data := []byte{0x36}
+	schema := "string"
+	dec, err := avro.NewDecoder(schema, bytes.NewReader(data))
+	require.NoError(t, err)
+
+	var i uint16
+	err = dec.Decode(&i)
+
+	assert.Error(t, err)
+}
+
 func TestDecoder_Int32(t *testing.T) {
 	defer ConfigTeardown()
 
@@ -170,6 +228,35 @@ func TestDecoder_Int32InvalidSchema(t *testing.T) {
 	assert.Error(t, err)
 }
 
+func TestDecoder_Uint32(t *testing.T) {
+	defer ConfigTeardown()
+
+	data := []byte{0x36}
+	schema := "long"
+	dec, err := avro.NewDecoder(schema, bytes.NewReader(data))
+	require.NoError(t, err)
+
+	var i uint32
+	err = dec.Decode(&i)
+
+	require.NoError(t, err)
+	assert.Equal(t, uint32(27), i)
+}
+
+func TestDecoder_Uint32InvalidSchema(t *testing.T) {
+	defer ConfigTeardown()
+
+	data := []byte{0x36}
+	schema := "int"
+	dec, err := avro.NewDecoder(schema, bytes.NewReader(data))
+	require.NoError(t, err)
+
+	var i uint32
+	err = dec.Decode(&i)
+
+	assert.Error(t, err)
+}
+
 func TestDecoder_Int64(t *testing.T) {
 	defer ConfigTeardown()
 
diff --git a/encoder_native_test.go b/encoder_native_test.go
index 4863439..a2f9858 100644
--- a/encoder_native_test.go
+++ b/encoder_native_test.go
@@ -105,6 +105,33 @@ func TestEncoder_Int8InvalidSchema(t *testing.T) {
 	assert.Error(t, err)
 }
 
+func TestEncoder_Uint8(t *testing.T) {
+	defer ConfigTeardown()
+
+	schema := "int"
+	buf := bytes.NewBuffer([]byte{})
+	enc, err := avro.NewEncoder(schema, buf)
+	require.NoError(t, err)
+
+	err = enc.Encode(uint8(27))
+
+	require.NoError(t, err)
+	assert.Equal(t, []byte{0x36}, buf.Bytes())
+}
+
+func TestEncoder_Uint8InvalidSchema(t *testing.T) {
+	defer ConfigTeardown()
+
+	schema := "string"
+	buf := bytes.NewBuffer([]byte{})
+	enc, err := avro.NewEncoder(schema, buf)
+	require.NoError(t, err)
+
+	err = enc.Encode(uint8(27))
+
+	assert.Error(t, err)
+}
+
 func TestEncoder_Int16(t *testing.T) {
 	defer ConfigTeardown()
 
@@ -132,6 +159,33 @@ func TestEncoder_Int16InvalidSchema(t *testing.T) {
 	assert.Error(t, err)
 }
 
+func TestEncoder_Uint16(t *testing.T) {
+	defer ConfigTeardown()
+
+	schema := "int"
+	buf := bytes.NewBuffer([]byte{})
+	enc, err := avro.NewEncoder(schema, buf)
+	require.NoError(t, err)
+
+	err = enc.Encode(uint16(27))
+
+	require.NoError(t, err)
+	assert.Equal(t, []byte{0x36}, buf.Bytes())
+}
+
+func TestEncoder_Uint16InvalidSchema(t *testing.T) {
+	defer ConfigTeardown()
+
+	schema := "string"
+	buf := bytes.NewBuffer([]byte{})
+	enc, err := avro.NewEncoder(schema, buf)
+	require.NoError(t, err)
+
+	err = enc.Encode(uint16(27))
+
+	assert.Error(t, err)
+}
+
 func TestEncoder_Int32(t *testing.T) {
 	defer ConfigTeardown()
 
@@ -159,6 +213,33 @@ func TestEncoder_Int32InvalidSchema(t *testing.T) {
 	assert.Error(t, err)
 }
 
+func TestEncoder_Uint32(t *testing.T) {
+	defer ConfigTeardown()
+
+	schema := "long"
+	buf := bytes.NewBuffer([]byte{})
+	enc, err := avro.NewEncoder(schema, buf)
+	require.NoError(t, err)
+
+	err = enc.Encode(uint32(27))
+
+	require.NoError(t, err)
+	assert.Equal(t, []byte{0x36}, buf.Bytes())
+}
+
+func TestEncoder_Uint32InvalidSchema(t *testing.T) {
+	defer ConfigTeardown()
+
+	schema := "int"
+	buf := bytes.NewBuffer([]byte{})
+	enc, err := avro.NewEncoder(schema, buf)
+	require.NoError(t, err)
+
+	err = enc.Encode(uint32(27))
+
+	assert.Error(t, err)
+}
+
 func TestEncoder_Int64(t *testing.T) {
 	defer ConfigTeardown()