Repository: jtablesaw/tablesaw (java)

Issue:
NullPointerException when reading json array with incomplete rows
Hello everyone, reading a list of objects with differing attribute sets results in a
```
Exception in thread "main" java.lang.NullPointerException
	at tech.tablesaw.io.json.JsonReader.convertArrayOfObjects(JsonReader.java:106)
	at tech.tablesaw.io.json.JsonReader.read(JsonReader.java:51)
	at tech.tablesaw.io.json.JsonReader.read(JsonReader.java:21)
	at tech.tablesaw.io.DataFrameReader.usingOptions(DataFrameReader.java:103)
```
In the previous code block in JsonReader, each field name in every row is added to the `colNames` set. However, afterwards at line 106, it is just assumed that the respective node has all the attributes stored in `colNames`, which leads to this exception being thrown.

Here is a minimal reproducible example. The expected behavior for me would be to treat A and C as missing values in the respective row in which they are actually missing. This occurred and is reproducible in tablesaw-json v0.34.2.:
```
String s = "[" +
    "{\"A\" : \"123\", \"B\" : \"456\"}," +
    "{\"B\" : \"789\", \"C\" : \"123\"}" +
    "]";

JsonReadOptions.Builder builder = JsonReadOptions.builderFromString(s);

try{
    JsonReadOptions options = builder.build();
    DataFrameReader dfr = Table.read();
    Table t = dfr.usingOptions(options);
} catch (IOException e){ }
```
Best,
David

PS: I like tablesaw a lot, it is one of the reasons why I can actually use java for my research projects. I am used to working with python/pandas and R data frames and the lack of such easy-access API in Java is incomprehensible to me. I'd like to contribute to this project, where possible.


Pull request:
Fix NPE when reading json array with incomplete rows
Thanks for contributing.

- [x] Tick to sign-off your agreement to the [Developer Certificate of Origin (DCO) 1.1](https://developercertificate.org)

## Description

Fixes #588 

Checks whether a json key exists before getting attempting to get it. If the key does not exist add missing a value.

## Testing
Yes

Patch:
diff --git a/core/src/main/java/tech/tablesaw/aggregate/AggregateFunctions.java b/core/src/main/java/tech/tablesaw/aggregate/AggregateFunctions.java
index 8eaaa849..a964bafd 100644
--- a/core/src/main/java/tech/tablesaw/aggregate/AggregateFunctions.java
+++ b/core/src/main/java/tech/tablesaw/aggregate/AggregateFunctions.java
@@ -203,7 +203,10 @@ public class AggregateFunctions {
         }
       };
 
-  /** A function that calculates the coefficient of variation (stdDev/mean) of the values in the column */
+  /**
+   * A function that calculates the coefficient of variation (stdDev/mean) of the values in the
+   * column
+   */
   public static final NumericAggregateFunction cv =
       new NumericAggregateFunction("CV") {
 
diff --git a/core/src/main/java/tech/tablesaw/aggregate/Summarizer.java b/core/src/main/java/tech/tablesaw/aggregate/Summarizer.java
index cac2bf64..874d12c5 100644
--- a/core/src/main/java/tech/tablesaw/aggregate/Summarizer.java
+++ b/core/src/main/java/tech/tablesaw/aggregate/Summarizer.java
@@ -14,6 +14,8 @@
 
 package tech.tablesaw.aggregate;
 
+import static tech.tablesaw.api.QuerySupport.numberColumn;
+
 import com.google.common.base.Preconditions;
 import com.google.common.collect.ArrayListMultimap;
 import java.util.ArrayList;
@@ -32,8 +34,6 @@ import tech.tablesaw.selection.Selection;
 import tech.tablesaw.table.StandardTableSliceGroup;
 import tech.tablesaw.table.TableSliceGroup;
 
-import static tech.tablesaw.api.QuerySupport.numberColumn;
-
 /**
  * Summarizes the data in a table, by applying functions to a subset of its columns.
  *
@@ -338,7 +338,7 @@ public class Summarizer {
     int withinGroupCount = 0;
     Row row = new Row(temp);
 
-    while(row.hasNext()) {
+    while (row.hasNext()) {
       row.next();
       if (withinGroupCount < step) {
         withinGroupCount++;
@@ -349,8 +349,7 @@ public class Summarizer {
         withinGroupCount = 1;
       }
     }
-    int lastGroupSize =
-            temp.where(numberColumn(GROUP_COL_TEMP_NAME).isEqualTo(groupId)).rowCount();
+    int lastGroupSize = temp.where(numberColumn(GROUP_COL_TEMP_NAME).isEqualTo(groupId)).rowCount();
     if (lastGroupSize < step) {
       temp = temp.dropWhere(numberColumn(GROUP_COL_TEMP_NAME).isEqualTo(groupId));
     }
diff --git a/core/src/main/java/tech/tablesaw/columns/AbstractColumn.java b/core/src/main/java/tech/tablesaw/columns/AbstractColumn.java
index bf2c36ff..f2656ed3 100644
--- a/core/src/main/java/tech/tablesaw/columns/AbstractColumn.java
+++ b/core/src/main/java/tech/tablesaw/columns/AbstractColumn.java
@@ -14,24 +14,8 @@
 
 package tech.tablesaw.columns;
 
-import java.time.Instant;
-import java.time.LocalDate;
-import java.time.LocalDateTime;
-import java.time.LocalTime;
-import java.util.function.Function;
-import tech.tablesaw.api.BooleanColumn;
 import tech.tablesaw.api.ColumnType;
-import tech.tablesaw.api.DateColumn;
-import tech.tablesaw.api.DateTimeColumn;
-import tech.tablesaw.api.DoubleColumn;
-import tech.tablesaw.api.FloatColumn;
-import tech.tablesaw.api.InstantColumn;
-import tech.tablesaw.api.IntColumn;
-import tech.tablesaw.api.LongColumn;
-import tech.tablesaw.api.ShortColumn;
 import tech.tablesaw.api.StringColumn;
-import tech.tablesaw.api.TextColumn;
-import tech.tablesaw.api.TimeColumn;
 
 /** Partial implementation of the {@link Column} interface */
 public abstract class AbstractColumn<T> implements Column<T> {
diff --git a/core/src/main/java/tech/tablesaw/table/Relation.java b/core/src/main/java/tech/tablesaw/table/Relation.java
index 4a803e45..35c76cc6 100644
--- a/core/src/main/java/tech/tablesaw/table/Relation.java
+++ b/core/src/main/java/tech/tablesaw/table/Relation.java
@@ -18,9 +18,7 @@ import it.unimi.dsi.fastutil.ints.IntArrays;
 import java.io.ByteArrayOutputStream;
 import java.util.ArrayList;
 import java.util.Arrays;
-import java.util.Iterator;
 import java.util.List;
-import java.util.function.Function;
 import java.util.stream.Collectors;
 import tech.tablesaw.api.BooleanColumn;
 import tech.tablesaw.api.CategoricalColumn;
diff --git a/core/src/main/java/tech/tablesaw/table/TableSlice.java b/core/src/main/java/tech/tablesaw/table/TableSlice.java
index 118bf112..43509503 100644
--- a/core/src/main/java/tech/tablesaw/table/TableSlice.java
+++ b/core/src/main/java/tech/tablesaw/table/TableSlice.java
@@ -14,7 +14,6 @@
 
 package tech.tablesaw.table;
 
-import it.unimi.dsi.fastutil.ints.IntIterable;
 import it.unimi.dsi.fastutil.ints.IntIterator;
 import java.util.ArrayList;
 import java.util.Iterator;
@@ -166,9 +165,7 @@ public class TableSlice extends Relation {
     return function.summarize(column.where(selection));
   }
 
-  /**
-   * Iterate of a copy of the table.
-   */
+  /** Iterate of a copy of the table. */
   @Override
   public Iterator<Row> iterator() {
 
diff --git a/json/src/main/java/tech/tablesaw/io/json/JsonReader.java b/json/src/main/java/tech/tablesaw/io/json/JsonReader.java
index fad26ed5..a17164c2 100644
--- a/json/src/main/java/tech/tablesaw/io/json/JsonReader.java
+++ b/json/src/main/java/tech/tablesaw/io/json/JsonReader.java
@@ -101,7 +101,11 @@ public class JsonReader implements DataReader<JsonReadOptions> {
     for (JsonNode node : flattenedJsonObj) {
       String[] arr = new String[columnNames.size()];
       for (int i = 0; i < columnNames.size(); i++) {
-        arr[i] = node.get(columnNames.get(i)).asText();
+        if(node.has(columnNames.get(i))) {
+          arr[i] = node.get(columnNames.get(i)).asText();
+        } else {
+          arr[i] = null;
+        }
       }
       dataRows.add(arr);
     }


Test patch:
diff --git a/core/src/test/java/tech/tablesaw/api/FloatColumnTest.java b/core/src/test/java/tech/tablesaw/api/FloatColumnTest.java
index 9a8f59fc..2e49443c 100644
--- a/core/src/test/java/tech/tablesaw/api/FloatColumnTest.java
+++ b/core/src/test/java/tech/tablesaw/api/FloatColumnTest.java
@@ -14,7 +14,6 @@
 
 package tech.tablesaw.api;
 
-import static org.junit.jupiter.api.Assertions.assertArrayEquals;
 import static org.junit.jupiter.api.Assertions.assertEquals;
 
 import org.junit.jupiter.api.Test;
diff --git a/core/src/test/java/tech/tablesaw/api/NumberColumnTest.java b/core/src/test/java/tech/tablesaw/api/NumberColumnTest.java
index c466568a..6cc66c28 100644
--- a/core/src/test/java/tech/tablesaw/api/NumberColumnTest.java
+++ b/core/src/test/java/tech/tablesaw/api/NumberColumnTest.java
@@ -468,6 +468,19 @@ public class NumberColumnTest {
     assertEquals(notUniques.length - 1, doubles.countUnique());
   }
 
+  @Test
+  public void testCountUniqueWithMissing() {
+    IntColumn ints = IntColumn.create("ints");
+    ints.append(1);
+    ints.append(2);
+    ints.append(3);
+    ints.appendMissing();
+
+    assertEquals(3, ints.countUnique());
+    assertEquals(3, ints.unique().size());
+  }
+
+
   @Test
   public void testUnique() {
     DoubleColumn doubles = DoubleColumn.create("doubles");
diff --git a/core/src/test/java/tech/tablesaw/api/StringColumnTest.java b/core/src/test/java/tech/tablesaw/api/StringColumnTest.java
index 1bd092ad..f06deb5b 100644
--- a/core/src/test/java/tech/tablesaw/api/StringColumnTest.java
+++ b/core/src/test/java/tech/tablesaw/api/StringColumnTest.java
@@ -663,4 +663,30 @@ class StringColumnTest {
     assertEquals(0, col.countUnique(), "Wrong number of unique values after Column.clear");
     assertEquals(0, col.countMissing(), "Wrong number of missing values after Column.clear");
   }
+
+  @Test
+  public void countUniqueWithDuplicates() {
+    StringColumn col = StringColumn.create("col1");
+    col.append("1");
+    col.append("1");
+    col.append("2");
+    col.append("2");
+    col.append("3");
+
+    assertEquals(3, col.countUnique());
+    assertEquals(3, col.unique().size());
+    assertEquals(3, col.asStringColumn().unique().size());
+  }
+
+  @Test
+  public void countUniqueWithMissing() {
+    StringColumn col1 = StringColumn.create("col1");
+    col1.append("1");
+    col1.append("2");
+    col1.append("3");
+    col1.appendMissing();
+
+    assertEquals(4, col1.countUnique());
+    assertEquals(4, col1.unique().size());
+  }
 }
diff --git a/core/src/test/java/tech/tablesaw/api/TableTest.java b/core/src/test/java/tech/tablesaw/api/TableTest.java
index 98b5767b..313442ce 100644
--- a/core/src/test/java/tech/tablesaw/api/TableTest.java
+++ b/core/src/test/java/tech/tablesaw/api/TableTest.java
@@ -39,8 +39,6 @@ import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Test;
 import tech.tablesaw.columns.Column;
 import tech.tablesaw.columns.dates.PackedLocalDate;
-import tech.tablesaw.columns.numbers.DoubleColumnType;
-import tech.tablesaw.columns.numbers.IntColumnType;
 import tech.tablesaw.io.csv.CsvReadOptions;
 
 public class TableTest {
diff --git a/core/src/test/java/tech/tablesaw/filters/TimeDependentFilteringTest.java b/core/src/test/java/tech/tablesaw/filters/TimeDependentFilteringTest.java
index 333e27a0..ebb499c7 100644
--- a/core/src/test/java/tech/tablesaw/filters/TimeDependentFilteringTest.java
+++ b/core/src/test/java/tech/tablesaw/filters/TimeDependentFilteringTest.java
@@ -117,7 +117,7 @@ public class TimeDependentFilteringTest {
       for (Row row : patientTable) {
         StringColumn concepts = patientTable.stringColumn("concept");
         DateColumn dates = patientTable.dateColumn("date");
-        if(row.getString("concept").equals(conceptZ)) {
+        if (row.getString("concept").equals(conceptZ)) {
           eventDates.add(row.getDate("date"));
         }
       }
diff --git a/json/src/test/java/tech/tablesaw/io/json/JsonReaderTest.java b/json/src/test/java/tech/tablesaw/io/json/JsonReaderTest.java
index e75cc4f7..041e753b 100644
--- a/json/src/test/java/tech/tablesaw/io/json/JsonReaderTest.java
+++ b/json/src/test/java/tech/tablesaw/io/json/JsonReaderTest.java
@@ -18,6 +18,7 @@ import static org.junit.jupiter.api.Assertions.assertEquals;
 
 import org.junit.jupiter.api.Test;
 import tech.tablesaw.api.ColumnType;
+import tech.tablesaw.api.IntColumn;
 import tech.tablesaw.api.Table;
 
 public class JsonReaderTest {
@@ -32,6 +33,7 @@ public class JsonReaderTest {
     assertEquals("Date", table.column(0).name());
     assertEquals("Value", table.column(1).name());
     assertEquals(ColumnType.LONG, table.columnTypes()[0]);
+    assertEquals(1453438800000L, table.column("Date").get(0));
   }
 
   @Test
@@ -55,4 +57,25 @@ public class JsonReaderTest {
     assertEquals("b.c", table.column(1).name());
     assertEquals(ColumnType.LONG, table.columnTypes()[0]);
   }
+
+  @Test
+  public void arrayOfRowsWithIncompleteIndexes() {
+    String json = "[" +
+      "{\"A\" : \"123\", \"B\" : \"456\"}," +
+      "{\"B\" : \"789\", \"C\" : \"123\"}" +
+      "]";
+
+    Table expected = Table.create(
+      IntColumn.create("A", new int[]{123, Integer.MIN_VALUE}),
+      IntColumn.create("B", new int[]{456, 789}),
+      IntColumn.create("C", new int[]{Integer.MIN_VALUE, 123})
+    );
+    Table actual = Table.read().string(json, "json");
+
+    assertEquals(ColumnType.INTEGER, actual.columnTypes()[0]);
+    assertEquals(expected.column("A").asList(), actual.column("A").asList());
+    assertEquals(expected.column("B").asList(), actual.column("B").asList());
+    assertEquals(expected.column("C").asList(), actual.column("C").asList());
+  }
+
 }