Bug: Pass all UTs in `external/avro` with Java 17
Status: Resolved | Priority: Major | Resolution: Fixed
Created: 20/Oct/21 14:23
Affects Version/s: 3.3.0

Run `mvn clean install -pl external/avro` with Java 17

 

 
{code:java}
Run completed in 43 seconds, 988 milliseconds.
Total number of tests run: 283
Suites: completed 14, aborted 0
Tests: succeeded 281, failed 2, canceled 0, ignored 2, pending 0
*** 2 TESTS FAILED ***
{code}
 
{code:java}
- support user provided non-nullable avro schema for nullable catalyst schema without any null record *** FAILED ***
  "Job aborted due to stage failure: Task 1 in stage 144.0 failed 1 times, most recent failure: Lost task 1.0 in stage 144.0 (TID 250) (localhost executor driver): org.apache.spark.SparkException: Task failed while writing rows.
  	at org.apache.spark.sql.errors.QueryExecutionErrors$.taskFailedWhileWritingRowsError(QueryExecutionErrors.scala:516)
  	at org.apache.spark.sql.execution.datasources.FileFormatWriter$.executeTask(FileFormatWriter.scala:345)
  	at org.apache.spark.sql.execution.datasources.FileFormatWriter$.$anonfun$write$20(FileFormatWriter.scala:252)
  	at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:90)
  	at org.apache.spark.scheduler.Task.run(Task.scala:136)
  	at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$3(Executor.scala:507)
  	at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:1468)
  	at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:510)
  	at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
  	at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
  	at java.base/java.lang.Thread.run(Thread.java:833)
  Caused by: org.apache.avro.file.DataFileWriter$AppendWriteException: java.lang.NullPointerException: Cannot invoke "Object.getClass()" because "datum" is null of string in string in field Name of test_schema in test_schema
  	at org.apache.avro.file.DataFileWriter.append(DataFileWriter.java:317)
  	at org.apache.spark.sql.avro.SparkAvroKeyRecordWriter.write(SparkAvroKeyOutputFormat.java:84)
  	at org.apache.spark.sql.avro.SparkAvroKeyRecordWriter.write(SparkAvroKeyOutputFormat.java:62)
  	at org.apache.spark.sql.avro.AvroOutputWriter.write(AvroOutputWriter.scala:84)
  	at org.apache.spark.sql.execution.datasources.SingleDirectoryDataWriter.write(FileFormatDataWriter.scala:175)
  	at org.apache.spark.sql.execution.datasources.FileFormatDataWriter.writeWithMetrics(FileFormatDataWriter.scala:85)
  	at org.apache.spark.sql.execution.datasources.FileFormatDataWriter.writeWithIterator(FileFormatDataWriter.scala:92)
  	at org.apache.spark.sql.execution.datasources.FileFormatWriter$.$anonfun$executeTask$1(FileFormatWriter.scala:328)
  	at org.apache.spark.util.Utils$.tryWithSafeFinallyAndFailureCallbacks(Utils.scala:1502)
  	at org.apache.spark.sql.execution.datasources.FileFormatWriter$.executeTask(FileFormatWriter.scala:335)
  	... 9 more
  Caused by: java.lang.NullPointerException: Cannot invoke "Object.getClass()" because "datum" is null of string in string in field Name of test_schema in test_schema
  	at org.apache.avro.generic.GenericDatumWriter.npe(GenericDatumWriter.java:184)
  	at org.apache.avro.reflect.ReflectDatumWriter.write(ReflectDatumWriter.java:160)
  	at org.apache.avro.generic.GenericDatumWriter.write(GenericDatumWriter.java:73)
  	at org.apache.avro.file.DataFileWriter.append(DataFileWriter.java:314)
  	... 18 more
  Caused by: java.lang.NullPointerException: Cannot invoke "Object.getClass()" because "datum" is null
  	at org.apache.avro.specific.SpecificDatumWriter.writeString(SpecificDatumWriter.java:68)
  	at org.apache.avro.generic.GenericDatumWriter.writeWithoutConversion(GenericDatumWriter.java:151)
  	at org.apache.avro.generic.GenericDatumWriter.write(GenericDatumWriter.java:83)
  	at org.apache.avro.reflect.ReflectDatumWriter.write(ReflectDatumWriter.java:158)
  	at org.apache.avro.generic.GenericDatumWriter.writeField(GenericDatumWriter.java:221)
  	at org.apache.avro.specific.SpecificDatumWriter.writeField(SpecificDatumWriter.java:101)
  	at org.apache.avro.reflect.ReflectDatumWriter.writeField(ReflectDatumWriter.java:184)
  	at org.apache.avro.generic.GenericDatumWriter.writeRecord(GenericDatumWriter.java:210)
  	at org.apache.avro.specific.SpecificDatumWriter.writeRecord(SpecificDatumWriter.java:83)
  	at org.apache.avro.generic.GenericDatumWriter.writeWithoutConversion(GenericDatumWriter.java:131)
  	at org.apache.avro.generic.GenericDatumWriter.write(GenericDatumWriter.java:83)
  	at org.apache.avro.reflect.ReflectDatumWriter.write(ReflectDatumWriter.java:158)
  	... 20 more
  
  Driver stacktrace:" did not contain "Caused by: java.lang.NullPointerException: null of string in string in field Name of test_schema in test_schema" (AvroSuite.scala:1074)
{code}