Repository: avro-kotlin/avro4k
Pull request #73 (2020-10-16 22:35:13)

Add generic default value support
Hi all,

Sorry I don't know what is the correct policy of contributing for the avro4k project. Example should I first raise an issue before opening a pull request. And another thing is that I'm not really that familiar with the kotlinx.serialization lib. 

But anyway can I suggest this small addition that would enable using the empty list default for more complex types than primitives. Sure the more correct solution would be adding full serialization for more complex default values but that probably would need more work.

Or this would be really helpful on my current project where we try to avoid using null default value for collections and just default those to empty list etc.


Patch:
diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml
index 333deed..2f22037 100644
--- a/.github/workflows/pr.yml
+++ b/.github/workflows/pr.yml
@@ -8,6 +8,7 @@ on:
 
 jobs:
    build:
+      runs-on: ubuntu-latest
       steps:
          -  name: Checkout the repo
             uses: actions/checkout@v2
diff --git a/src/main/kotlin/com/sksamuel/avro4k/Avro.kt b/src/main/kotlin/com/sksamuel/avro4k/Avro.kt
index 4f94e39..e7239b4 100644
--- a/src/main/kotlin/com/sksamuel/avro4k/Avro.kt
+++ b/src/main/kotlin/com/sksamuel/avro4k/Avro.kt
@@ -2,17 +2,34 @@ package com.sksamuel.avro4k
 
 import com.sksamuel.avro4k.decoder.RootRecordDecoder
 import com.sksamuel.avro4k.encoder.RootRecordEncoder
-import com.sksamuel.avro4k.io.*
+import com.sksamuel.avro4k.io.AvroBinaryInputStream
+import com.sksamuel.avro4k.io.AvroBinaryOutputStream
+import com.sksamuel.avro4k.io.AvroDataInputStream
+import com.sksamuel.avro4k.io.AvroDataOutputStream
+import com.sksamuel.avro4k.io.AvroFormat
+import com.sksamuel.avro4k.io.AvroInputStream
+import com.sksamuel.avro4k.io.AvroJsonInputStream
+import com.sksamuel.avro4k.io.AvroJsonOutputStream
+import com.sksamuel.avro4k.io.AvroOutputStream
 import com.sksamuel.avro4k.schema.DefaultNamingStrategy
 import com.sksamuel.avro4k.schema.schemaFor
 import com.sksamuel.avro4k.serializer.UUIDSerializer
-import kotlinx.serialization.*
+import kotlinx.serialization.BinaryFormat
+import kotlinx.serialization.DeserializationStrategy
+import kotlinx.serialization.ExperimentalSerializationApi
+import kotlinx.serialization.SerialFormat
+import kotlinx.serialization.SerializationException
+import kotlinx.serialization.SerializationStrategy
 import kotlinx.serialization.modules.EmptySerializersModule
 import kotlinx.serialization.modules.SerializersModule
 import org.apache.avro.Schema
 import org.apache.avro.file.CodecFactory
 import org.apache.avro.generic.GenericRecord
-import java.io.*
+import java.io.ByteArrayInputStream
+import java.io.ByteArrayOutputStream
+import java.io.File
+import java.io.InputStream
+import java.io.OutputStream
 import java.nio.ByteBuffer
 import java.nio.file.Files
 import java.nio.file.Path
@@ -52,6 +69,8 @@ class AvroInputStreamBuilder<T>(private val converter: (Any) -> T) {
                AvroDataInputStream(source, converter, writerSchema, readerSchema)
             writerSchema != null ->
                AvroDataInputStream(source, converter, writerSchema, readerSchema)
+            readerSchema != null ->
+               AvroDataInputStream(source,converter,null, readerSchema)
             else ->
                AvroDataInputStream(source, converter, null, null)
          }
@@ -219,4 +238,6 @@ class Avro(override val serializersModule: SerializersModule = EmptySerializersM
          DefaultNamingStrategy
       ).schema()
    }
+
+
 }
\ No newline at end of file
diff --git a/src/main/kotlin/com/sksamuel/avro4k/SerialDescriptor.kt b/src/main/kotlin/com/sksamuel/avro4k/SerialDescriptor.kt
index 3e22762..c0abd7a 100644
--- a/src/main/kotlin/com/sksamuel/avro4k/SerialDescriptor.kt
+++ b/src/main/kotlin/com/sksamuel/avro4k/SerialDescriptor.kt
@@ -1,11 +1,10 @@
 package com.sksamuel.avro4k
 
 import kotlinx.serialization.ExperimentalSerializationApi
-import kotlinx.serialization.KSerializer
-import kotlinx.serialization.builtins.ListSerializer
-import kotlinx.serialization.builtins.nullable
-import kotlinx.serialization.builtins.serializer
-import kotlinx.serialization.descriptors.*
+import kotlinx.serialization.descriptors.PolymorphicKind
+import kotlinx.serialization.descriptors.SerialDescriptor
+import kotlinx.serialization.descriptors.SerialKind
+import kotlinx.serialization.descriptors.elementDescriptors
 
 @ExperimentalSerializationApi
 fun SerialDescriptor.leavesOfSealedClasses() : List<SerialDescriptor> {
@@ -14,26 +13,4 @@ fun SerialDescriptor.leavesOfSealedClasses() : List<SerialDescriptor> {
    } else {
       listOf(this)
    }
-}
-
-@ExperimentalSerializationApi
-fun SerialDescriptor.serializer() : KSerializer<out Any?> {
-   val notNullableSerializer = when(kind){
-      PrimitiveKind.BOOLEAN -> Boolean.serializer()
-      PrimitiveKind.INT -> Int.serializer()
-      PrimitiveKind.LONG -> Long.serializer()
-      PrimitiveKind.FLOAT -> Float.serializer()
-      PrimitiveKind.BOOLEAN -> Boolean.serializer()
-      PrimitiveKind.BYTE -> Byte.serializer()
-      PrimitiveKind.SHORT -> Short.serializer()
-      PrimitiveKind.STRING -> String.serializer()
-      StructureKind.LIST -> ListSerializer(this.elementDescriptors.single().serializer())
-      else -> TODO("only implemented primitive types for now")
-}
-   return if(this.isNullable){
-      notNullableSerializer.nullable
-   }else{
-      notNullableSerializer
-   }
-
 }
\ No newline at end of file
diff --git a/src/main/kotlin/com/sksamuel/avro4k/schema/ClassSchemaFor.kt b/src/main/kotlin/com/sksamuel/avro4k/schema/ClassSchemaFor.kt
index 67a7741..a5e20f6 100644
--- a/src/main/kotlin/com/sksamuel/avro4k/schema/ClassSchemaFor.kt
+++ b/src/main/kotlin/com/sksamuel/avro4k/schema/ClassSchemaFor.kt
@@ -1,21 +1,29 @@
 package com.sksamuel.avro4k.schema
 
-import com.sksamuel.avro4k.*
+import com.sksamuel.avro4k.AnnotationExtractor
+import com.sksamuel.avro4k.Avro
+import com.sksamuel.avro4k.AvroProp
+import com.sksamuel.avro4k.RecordNaming
 import kotlinx.serialization.ExperimentalSerializationApi
-import kotlinx.serialization.SerializationException
-import kotlinx.serialization.descriptors.PrimitiveKind
 import kotlinx.serialization.descriptors.SerialDescriptor
-import kotlinx.serialization.descriptors.StructureKind
 import kotlinx.serialization.json.Json
+import kotlinx.serialization.json.JsonArray
+import kotlinx.serialization.json.JsonElement
+import kotlinx.serialization.json.JsonNull
+import kotlinx.serialization.json.JsonObject
+import kotlinx.serialization.json.JsonPrimitive
+import kotlinx.serialization.json.boolean
+import kotlinx.serialization.json.booleanOrNull
 import kotlinx.serialization.modules.SerializersModule
 import org.apache.avro.JsonProperties
 import org.apache.avro.Schema
 import org.apache.avro.SchemaBuilder
 
 @ExperimentalSerializationApi
-class ClassSchemaFor(private val descriptor: SerialDescriptor,
-                     private val namingStrategy: NamingStrategy,
-                     private val serializersModule: SerializersModule
+class ClassSchemaFor(
+   private val descriptor: SerialDescriptor,
+   private val namingStrategy: NamingStrategy,
+   private val serializersModule: SerializersModule
 ) : SchemaFor {
 
    private val entityAnnotations = AnnotationExtractor(descriptor.annotations)
@@ -52,12 +60,18 @@ class ClassSchemaFor(private val descriptor: SerialDescriptor,
       return record
    }
 
+
    private fun buildField(index: Int): Schema.Field {
 
       val fieldDescriptor = descriptor.getElementDescriptor(index)
       val annos = AnnotationExtractor(descriptor.getElementAnnotations(index))
       val fieldNaming = RecordNaming(descriptor, index)
-      val schema = schemaFor(serializersModule, fieldDescriptor, descriptor.getElementAnnotations(index), namingStrategy)
+      val schema = schemaFor(
+         serializersModule,
+         fieldDescriptor,
+         descriptor.getElementAnnotations(index),
+         namingStrategy
+      )
          .schema()
 
       // if we have annotated the field @AvroFixed then we override the type and change it to a Fixed schema
@@ -92,25 +106,11 @@ class ClassSchemaFor(private val descriptor: SerialDescriptor,
          else -> schemaOrFixed.overrideNamespace(ns)
       }
 
-
       val default: Any? = annos.default()?.let {
-         if (it == Avro.NULL) {
-            Schema.Field.NULL_DEFAULT_VALUE
-         } else {
-            when (fieldDescriptor.kind) {
-               PrimitiveKind.INT -> it.toInt()
-               PrimitiveKind.LONG -> it.toLong()
-               PrimitiveKind.FLOAT -> it.toFloat()
-               PrimitiveKind.BOOLEAN -> it.toBoolean()
-               PrimitiveKind.BYTE -> it.toByte()
-               PrimitiveKind.SHORT -> it.toShort()
-               PrimitiveKind.STRING -> it
-               StructureKind.LIST ->
-                  decodeJsonDefaultAsList(fieldDescriptor
-                  , it)
-
-               else -> throw IllegalArgumentException("Cannot use a default value for type ${fieldDescriptor.kind}")
-            }
+         when {
+             it == Avro.NULL -> Schema.Field.NULL_DEFAULT_VALUE
+             schemaWithResolvedNamespace.extractNonNull().type in listOf(Schema.Type.FIXED, Schema.Type.BYTES, Schema.Type.STRING) -> it
+             else -> json.parseToJsonElement(it).convertToAvroDefault()
          }
       }
 
@@ -122,14 +122,25 @@ class ClassSchemaFor(private val descriptor: SerialDescriptor,
       return field
    }
 
-   private fun  decodeJsonDefaultAsList(listFieldDescriptor: SerialDescriptor, jsonString: String): List<Any> = try {
-      // the list entries will be parsed according to their kind
-      val decodedValue = json.decodeFromString(listFieldDescriptor.serializer(), jsonString)
-      (decodedValue as? List<*>)?.map { it?:JsonProperties.NULL_VALUE } ?: error("Serializer of an array field descriptor did not return a List in its deserialized form.")
-   } catch (se: SerializationException) {
-      throw IllegalArgumentException("Cannot use default value $jsonString. ${se.message}",se)
+   private fun JsonElement.convertToAvroDefault() : Any{
+      return when(this){
+         is JsonNull -> JsonProperties.NULL_VALUE
+         is JsonObject -> this.map { Pair(it.key,it.value.convertToAvroDefault()) }.toMap()
+         is JsonArray -> this.map { it.convertToAvroDefault() }.toList()
+         is JsonPrimitive -> when {
+             this.isString -> this.content
+             this.booleanOrNull != null -> this.boolean
+             else -> {
+                 val number = this.content.toBigDecimal()
+                 if(number.scale() <= 0){
+                    number.toBigInteger()
+                 }else{
+                    number
+                 }
+             }
+         }
+      }
    }
 
 
 }
-


Test patch:
diff --git a/src/test/kotlin/com/sksamuel/avro4k/decoder/AvroDefaultValuesDecoderTest.kt b/src/test/kotlin/com/sksamuel/avro4k/decoder/AvroDefaultValuesDecoderTest.kt
new file mode 100644
index 0000000..9d02a41
--- /dev/null
+++ b/src/test/kotlin/com/sksamuel/avro4k/decoder/AvroDefaultValuesDecoderTest.kt
@@ -0,0 +1,78 @@
+package com.sksamuel.avro4k.decoder
+
+import com.sksamuel.avro4k.Avro
+import com.sksamuel.avro4k.AvroDefault
+import com.sksamuel.avro4k.AvroName
+import com.sksamuel.avro4k.ScalePrecision
+import com.sksamuel.avro4k.io.AvroFormat
+import com.sksamuel.avro4k.serializer.BigDecimalSerializer
+import io.kotest.core.spec.style.FunSpec
+import io.kotest.matchers.collections.shouldBeEmpty
+import io.kotest.matchers.collections.shouldContainExactly
+import io.kotest.matchers.nulls.shouldBeNull
+import io.kotest.matchers.shouldBe
+import kotlinx.serialization.Serializable
+import kotlinx.serialization.encodeToByteArray
+import org.apache.avro.generic.GenericData
+import java.math.BigDecimal
+
+@Serializable
+data class FooElement(
+   val content : String
+)
+
+@Serializable
+@AvroName("container")
+data class ContainerWithoutDefaultFields(
+   val name : String
+)
+@Serializable
+@AvroName("container")
+data class ContainerWithDefaultFields(
+   val name : String,
+   @AvroDefault("hello")
+   val strDefault : String,
+   @AvroDefault("1")
+   val intDefault : Int,
+   @AvroDefault("true")
+   val booleanDefault : Boolean,
+   @AvroDefault("1.23")
+   val doubleDefault : Double,
+   @AvroDefault(Avro.NULL)
+   val nullableStr : String?,
+   @AvroDefault("""{"content":"foo"}""")
+   val foo : FooElement,
+   @AvroDefault("[]")
+   val emptyFooList : List<FooElement>,
+   @AvroDefault("""[{"content":"bar"}]""")
+   val filledFooList : List<FooElement>,
+   @AvroDefault("\u0000")
+   @ScalePrecision(0,10)
+   @Serializable(BigDecimalSerializer::class)
+   val bigDecimal : BigDecimal
+)
+
+class AvroDefaultValuesDecoderTest : FunSpec({
+   test("test default values correctly decoded") {
+      val name = "abc"
+      val schema = Avro.default.schema(ContainerWithoutDefaultFields.serializer())
+      val record = GenericData.Record(schema)
+      record.put("name", name)
+
+      val byteArray = Avro.default.encodeToByteArray(ContainerWithoutDefaultFields("abc"))
+      val deserialized = Avro.default.openInputStream(ContainerWithDefaultFields.serializer()){
+         format = AvroFormat.DataFormat
+         readerSchema = Avro.default.schema(ContainerWithDefaultFields.serializer())
+      }.from(byteArray).nextOrThrow()
+      deserialized.name.shouldBe("abc")
+      deserialized.strDefault.shouldBe("hello")
+      deserialized.intDefault.shouldBe(1)
+      deserialized.booleanDefault.shouldBe(true)
+      deserialized.doubleDefault.shouldBe(1.23)
+      deserialized.nullableStr.shouldBeNull()
+      deserialized.foo.content.shouldBe("foo")
+      deserialized.emptyFooList.shouldBeEmpty()
+      deserialized.filledFooList.shouldContainExactly(FooElement("bar"))
+      deserialized.bigDecimal.shouldBe(BigDecimal.ZERO)
+   }
+})
\ No newline at end of file
diff --git a/src/test/kotlin/com/sksamuel/avro4k/schema/AvroDefaultSchemaTest.kt b/src/test/kotlin/com/sksamuel/avro4k/schema/AvroDefaultSchemaTest.kt
index a2efba5..6acb916 100644
--- a/src/test/kotlin/com/sksamuel/avro4k/schema/AvroDefaultSchemaTest.kt
+++ b/src/test/kotlin/com/sksamuel/avro4k/schema/AvroDefaultSchemaTest.kt
@@ -2,11 +2,16 @@ package com.sksamuel.avro4k.schema
 
 import com.sksamuel.avro4k.Avro
 import com.sksamuel.avro4k.AvroDefault
+import com.sksamuel.avro4k.serializer.BigDecimalSerializer
 import io.kotest.assertions.throwables.shouldThrow
-import io.kotest.matchers.shouldBe
 import io.kotest.core.spec.style.FunSpec
+import io.kotest.matchers.shouldBe
 import kotlinx.serialization.Serializable
+import org.apache.avro.AvroTypeException
+import org.apache.avro.Conversions
+import java.math.BigDecimal
 
+@Suppress("BlockingMethodInNonBlockingContext")
 class AvroDefaultSchemaTest : FunSpec() {
    init {
       test("schema for data class with @AvroDefault should include default value as a string") {
@@ -27,12 +32,25 @@ class AvroDefaultSchemaTest : FunSpec() {
          schema.toString(true) shouldBe expected.toString(true)
       }
 
+      test("schema for data class with @AvroDefault should include default value as a BigDecimal") {
+         val expected = org.apache.avro.Schema.Parser().parse(javaClass.getResourceAsStream("/avro_default_annotation_big_decimal.json"))
+         val conversion = Conversions.DecimalConversion()
+         val schema = Avro.default.schema(BarDecimal.serializer())
+         schema.toString(true) shouldBe expected.toString(true)
+      }
+
       test("schema for data class with @AvroDefault should include default value as a list") {
          val expected = org.apache.avro.Schema.Parser().parse(javaClass.getResourceAsStream("/avro_default_annotation_list.json"))
          val schema = Avro.default.schema(BarList.serializer())
          schema.toString(true) shouldBe expected.toString(true)
       }
 
+      test("schema for data class with @AvroDefault should include default value as a list with a record element type") {
+         val expected = org.apache.avro.Schema.Parser().parse(javaClass.getResourceAsStream("/avro_default_annotation_list_of_records.json"))
+         val schema = Avro.default.schema(BarListOfElements.serializer())
+         schema.toString(true) shouldBe expected.toString(true)
+      }
+
       test("schema for data class with @AvroDefault should include default value as an array") {
          val expected = org.apache.avro.Schema.Parser().parse(javaClass.getResourceAsStream("/avro_default_annotation_array.json"))
          val schema = Avro.default.schema(BarArray.serializer())
@@ -46,9 +64,8 @@ class AvroDefaultSchemaTest : FunSpec() {
       }
 
       test("schema for data class with @AvroDefault should throw error when array type does not match default value type") {
-         shouldThrow<IllegalArgumentException> { Avro.default.schema(BarInvalidArrayType.serializer()) }
-         shouldThrow<NotImplementedError> { Avro.default.toRecord(BarInvalidNonPrimitiveType.serializer(), BarInvalidNonPrimitiveType()) }
-         shouldThrow<IllegalArgumentException> { Avro.default.toRecord(BarInvalidNonArrayType.serializer(), BarInvalidNonArrayType()) }
+         shouldThrow<AvroTypeException> { Avro.default.schema(BarInvalidArrayType.serializer()) }
+         shouldThrow<AvroTypeException> { Avro.default.toRecord(BarInvalidNonArrayType.serializer(), BarInvalidNonArrayType()) }
       }
    }
 }
@@ -87,6 +104,21 @@ data class BarFloat(
    val c: Float?
 )
 
+@Serializable
+data class BarDecimal(
+   @Serializable(BigDecimalSerializer::class)
+   val a: BigDecimal,
+   @Serializable(BigDecimalSerializer::class)
+   @AvroDefault("\u0000")
+   val b: BigDecimal,
+   @Serializable(BigDecimalSerializer::class)
+   @AvroDefault(Avro.NULL)
+   val nullableString: BigDecimal?,
+   @Serializable(BigDecimalSerializer::class)
+   @AvroDefault("\u0000")
+   val c: BigDecimal?
+)
+
 @Serializable
 data class BarSet(
    @AvroDefault("[]")
@@ -123,6 +155,18 @@ data class BarList(
    @AvroDefault("""[null]""")
    val defaultStringListWithNullableTypes : List<String?>
 )
+
+@Serializable
+data class FooElement(val value: String)
+
+@Serializable
+data class BarListOfElements(
+   @AvroDefault("[]")
+   val defaultEmptyListOfRecords: List<FooElement>,
+   @AvroDefault("""[{"value":"foo"}]""")
+   val defaultListWithOneValue : List<FooElement>
+)
+
 @Suppress("ArrayInDataClass")
 @Serializable
 data class BarArray(
@@ -149,15 +193,6 @@ data class BarInvalidArrayType(
    val defaultFloatArrayWith2Defaults: List<Float>
 )
 
-@Serializable
-class FooBar
-
-@Serializable
-data class BarInvalidNonPrimitiveType(
-   @AvroDefault("test-value")
-   val defaultBarArrayWithNonWorkingDefaults: List<FooBar> = ArrayList()
-)
-
 @Serializable
 data class BarInvalidNonArrayType(
    @AvroDefault("{}")
diff --git a/src/test/resources/avro_default_annotation_big_decimal.json b/src/test/resources/avro_default_annotation_big_decimal.json
new file mode 100644
index 0000000..e17abd5
--- /dev/null
+++ b/src/test/resources/avro_default_annotation_big_decimal.json
@@ -0,0 +1,51 @@
+{
+   "type": "record",
+   "name": "BarDecimal",
+   "namespace": "com.sksamuel.avro4k.schema",
+   "fields": [
+      {
+         "name": "a",
+         "type": {
+            "type": "bytes",
+            "logicalType": "decimal",
+            "precision": 8,
+            "scale": 2
+         }
+      },
+      {
+         "name": "b",
+         "type": {
+            "type": "bytes",
+            "logicalType": "decimal",
+            "precision": 8,
+            "scale": 2
+         },
+         "default": "\u0000"
+      },
+      {
+         "name": "nullableString",
+         "type": ["null",
+            {
+               "type": "bytes",
+               "logicalType": "decimal",
+               "precision": 8,
+               "scale": 2
+            }
+         ],
+         "default" : null
+      },
+      {
+         "name": "c",
+         "type": [
+            {
+               "type": "bytes",
+               "logicalType": "decimal",
+               "precision": 8,
+               "scale": 2
+            }
+         ,"null"
+         ],
+         "default": "\u0000"
+      }
+   ]
+}
diff --git a/src/test/resources/avro_default_annotation_list_of_records.json b/src/test/resources/avro_default_annotation_list_of_records.json
new file mode 100644
index 0000000..7b3a746
--- /dev/null
+++ b/src/test/resources/avro_default_annotation_list_of_records.json
@@ -0,0 +1,32 @@
+{
+   "type": "record",
+   "name": "BarListOfElements",
+   "namespace": "com.sksamuel.avro4k.schema",
+   "fields": [
+      {
+         "name": "defaultEmptyListOfRecords",
+         "type": {
+            "type": "array",
+            "items": {
+               "type": "record",
+               "name": "FooElement",
+               "fields": [
+                  {
+                     "name": "value",
+                     "type": "string"
+                  }
+               ]
+            }
+         },
+         "default": []
+      }, {
+         "name" : "defaultListWithOneValue",
+         "type" : {
+            "type" : "array",
+            "items" : "FooElement"
+         },
+         "default" : [ {
+            "value" : "foo"
+         } ]
+      } ]
+}