Repository: jtablesaw/tablesaw
Pull request #975 (2021-08-20 21:58:54)

Fix issue #974
Missing values in numeric columns could not be formatted for print or writing to files

Thanks for contributing.

- [x] Tick to sign-off your agreement to the [Developer Certificate of Origin (DCO) 1.1](https://developercertificate.org)

## Description

The getString() methods in numeric columns were modified as they were short-circuting the handling of missing values. NumberColumnFormatter was modified to have a ColumnType field so it could be do the proper missing value check.

## Testing

Did you add a unit test? Yes



Patch:
diff --git a/core/src/main/java/tech/tablesaw/api/DoubleColumn.java b/core/src/main/java/tech/tablesaw/api/DoubleColumn.java
index c8c373cd..3112662f 100644
--- a/core/src/main/java/tech/tablesaw/api/DoubleColumn.java
+++ b/core/src/main/java/tech/tablesaw/api/DoubleColumn.java
@@ -38,9 +38,6 @@ public class DoubleColumn extends NumberColumn<DoubleColumn, Double>
   @Override
   public String getString(int row) {
     final double value = getDouble(row);
-    if (DoubleColumnType.valueIsMissing(value)) {
-      return "";
-    }
     return String.valueOf(getPrintFormatter().format(value));
   }
 
diff --git a/core/src/main/java/tech/tablesaw/api/FloatColumn.java b/core/src/main/java/tech/tablesaw/api/FloatColumn.java
index c45f52c7..d017cccc 100644
--- a/core/src/main/java/tech/tablesaw/api/FloatColumn.java
+++ b/core/src/main/java/tech/tablesaw/api/FloatColumn.java
@@ -25,10 +25,7 @@ public class FloatColumn extends NumberColumn<FloatColumn, Float> {
   @Override
   public String getString(int row) {
     final float value = getFloat(row);
-    if (FloatColumnType.valueIsMissing(value)) {
-      return "";
-    }
-    return String.valueOf(getPrintFormatter().format(value));
+    return getPrintFormatter().format(value);
   }
 
   public static FloatColumn create(String name) {
diff --git a/core/src/main/java/tech/tablesaw/api/IntColumn.java b/core/src/main/java/tech/tablesaw/api/IntColumn.java
index 589c26c0..211a130c 100644
--- a/core/src/main/java/tech/tablesaw/api/IntColumn.java
+++ b/core/src/main/java/tech/tablesaw/api/IntColumn.java
@@ -274,9 +274,6 @@ public class IntColumn extends NumberColumn<IntColumn, Integer>
   @Override
   public String getString(final int row) {
     final int value = getInt(row);
-    if (IntColumnType.valueIsMissing(value)) {
-      return "";
-    }
     return String.valueOf(getPrintFormatter().format(value));
   }
 
diff --git a/core/src/main/java/tech/tablesaw/api/LongColumn.java b/core/src/main/java/tech/tablesaw/api/LongColumn.java
index e1bc5a6f..d4062c4f 100644
--- a/core/src/main/java/tech/tablesaw/api/LongColumn.java
+++ b/core/src/main/java/tech/tablesaw/api/LongColumn.java
@@ -75,9 +75,6 @@ public class LongColumn extends NumberColumn<LongColumn, Long> implements Catego
   @Override
   public String getString(final int row) {
     final long value = getLong(row);
-    if (LongColumnType.valueIsMissing(value)) {
-      return "";
-    }
     return getPrintFormatter().format(value);
   }
 
diff --git a/core/src/main/java/tech/tablesaw/api/NumberColumn.java b/core/src/main/java/tech/tablesaw/api/NumberColumn.java
index 4bcb5692..0fa5ed5b 100644
--- a/core/src/main/java/tech/tablesaw/api/NumberColumn.java
+++ b/core/src/main/java/tech/tablesaw/api/NumberColumn.java
@@ -61,6 +61,7 @@ public abstract class NumberColumn<C extends NumberColumn<C, T>, T extends Numbe
   @Override
   public void setPrintFormatter(final NumberColumnFormatter formatter) {
     this.printFormatter = formatter;
+    formatter.setColumnType(type());
   }
 
   protected NumberColumnFormatter getPrintFormatter() {
diff --git a/core/src/main/java/tech/tablesaw/api/ShortColumn.java b/core/src/main/java/tech/tablesaw/api/ShortColumn.java
index 1711c291..4aef0267 100644
--- a/core/src/main/java/tech/tablesaw/api/ShortColumn.java
+++ b/core/src/main/java/tech/tablesaw/api/ShortColumn.java
@@ -268,9 +268,6 @@ public class ShortColumn extends NumberColumn<ShortColumn, Short>
   @Override
   public String getString(final int row) {
     final short value = getShort(row);
-    if (ShortColumnType.valueIsMissing(value)) {
-      return "";
-    }
     return String.valueOf(getPrintFormatter().format(value));
   }
 
diff --git a/core/src/main/java/tech/tablesaw/columns/Column.java b/core/src/main/java/tech/tablesaw/columns/Column.java
index 3ad45f60..c33b367c 100644
--- a/core/src/main/java/tech/tablesaw/columns/Column.java
+++ b/core/src/main/java/tech/tablesaw/columns/Column.java
@@ -78,7 +78,7 @@ public interface Column<T> extends Iterable<T>, Comparator<T> {
   ColumnType type();
 
   /**
-   * Returns the parser used by {@link #appendCell()}.
+   * Returns the parser used by {@link #appendCell(String)} ()}.
    *
    * @return {@link AbstractColumnParser}
    */
@@ -571,6 +571,7 @@ public interface Column<T> extends Iterable<T>, Comparator<T> {
 
   Column<T> set(int row, T value);
 
+  @SuppressWarnings("unchecked")
   default Column<T> set(int row, String stringValue, AbstractColumnParser<?> parser) {
     AbstractColumnParser<T> typedParser = (AbstractColumnParser<T>) parser;
     return set(row, typedParser.parse(stringValue));
@@ -620,7 +621,7 @@ public interface Column<T> extends Iterable<T>, Comparator<T> {
   Column<T> setName(String name);
 
   /**
-   * Sets the parser used by {@link #appendCell()}
+   * Sets the parser used by {@link #appendCell(String)}
    *
    * @param parser a column parser that converts text input to the column data type
    * @return this Column to allow method chaining
diff --git a/core/src/main/java/tech/tablesaw/columns/numbers/NumberColumnFormatter.java b/core/src/main/java/tech/tablesaw/columns/numbers/NumberColumnFormatter.java
index 45877691..f8ba4943 100644
--- a/core/src/main/java/tech/tablesaw/columns/numbers/NumberColumnFormatter.java
+++ b/core/src/main/java/tech/tablesaw/columns/numbers/NumberColumnFormatter.java
@@ -4,11 +4,13 @@ import java.text.DecimalFormat;
 import java.text.DecimalFormatSymbols;
 import java.text.NumberFormat;
 import java.util.Locale;
+import tech.tablesaw.api.ColumnType;
 import tech.tablesaw.columns.ColumnFormatter;
 
 public class NumberColumnFormatter extends ColumnFormatter {
 
   private final NumberFormat format;
+  private ColumnType columnType;
 
   public static NumberColumnFormatter percent(int fractionalDigits) {
     NumberFormat format = NumberFormat.getPercentInstance();
@@ -82,6 +84,14 @@ public class NumberColumnFormatter extends ColumnFormatter {
     this.format = null;
   }
 
+  public void setColumnType(ColumnType columnType) {
+    this.columnType = columnType;
+  }
+
+  public NumberFormat getFormat() {
+    return format;
+  }
+
   public String format(long value) {
     if (isMissingValue(value)) {
       return getMissingString();
@@ -92,6 +102,36 @@ public class NumberColumnFormatter extends ColumnFormatter {
     return format.format(value);
   }
 
+  public String format(int value) {
+    if (isMissingValue(value)) {
+      return getMissingString();
+    }
+    if (format == null) {
+      return String.valueOf(value);
+    }
+    return format.format(value);
+  }
+
+  public String format(short value) {
+    if (isMissingValue(value)) {
+      return getMissingString();
+    }
+    if (format == null) {
+      return String.valueOf(value);
+    }
+    return format.format(value);
+  }
+
+  public String format(float value) {
+    if (isMissingValue(value)) {
+      return getMissingString();
+    }
+    if (format == null) {
+      return String.valueOf(value);
+    }
+    return format.format(value);
+  }
+
   public String format(double value) {
     if (isMissingValue(value)) {
       return getMissingString();
@@ -114,6 +154,45 @@ public class NumberColumnFormatter extends ColumnFormatter {
   }
 
   private boolean isMissingValue(double value) {
-    return value != value;
+    if (columnType.equals(ColumnType.DOUBLE)) {
+      return DoubleColumnType.valueIsMissing(value);
+    } else {
+      throw new RuntimeException("Unhandled column type in NumberColumnFormatter: " + columnType);
+    }
+  }
+
+  private boolean isMissingValue(float value) {
+    if (columnType.equals(ColumnType.FLOAT)) {
+      return FloatColumnType.valueIsMissing(value);
+    } else {
+      throw new RuntimeException("Unhandled column type in NumberColumnFormatter: " + columnType);
+    }
+  }
+
+  private boolean isMissingValue(int value) {
+    if (columnType.equals(ColumnType.INTEGER)) {
+      return IntColumnType.valueIsMissing(value);
+    }
+    if (columnType.equals(ColumnType.SHORT)) {
+      return ShortColumnType.valueIsMissing(value);
+    } else {
+      throw new RuntimeException("Unhandled column type in NumberColumnFormatter: " + columnType);
+    }
+  }
+
+  private boolean isMissingValue(short value) {
+    if (columnType.equals(ColumnType.SHORT)) {
+      return ShortColumnType.valueIsMissing(value);
+    } else {
+      throw new RuntimeException("Unhandled column type in NumberColumnFormatter: " + columnType);
+    }
+  }
+
+  private boolean isMissingValue(long value) {
+    if (columnType.equals(ColumnType.LONG)) {
+      return LongColumnType.valueIsMissing(value);
+    } else {
+      throw new RuntimeException("Unhandled column type in NumberColumnFormatter: " + columnType);
+    }
   }
 }


Test patch:
diff --git a/core/src/test/java/tech/tablesaw/columns/numbers/NumberColumnFormatterTest.java b/core/src/test/java/tech/tablesaw/columns/numbers/NumberColumnFormatterTest.java
index e17c6843..952a85df 100644
--- a/core/src/test/java/tech/tablesaw/columns/numbers/NumberColumnFormatterTest.java
+++ b/core/src/test/java/tech/tablesaw/columns/numbers/NumberColumnFormatterTest.java
@@ -3,12 +3,15 @@ package tech.tablesaw.columns.numbers;
 import static org.junit.jupiter.api.Assertions.assertEquals;
 
 import org.junit.jupiter.api.Test;
+import tech.tablesaw.api.ColumnType;
 
 public class NumberColumnFormatterTest {
 
   @Test
   public void testFormatLong() {
     long value = 1588838400007002844L;
-    assertEquals(Long.toString(value), NumberColumnFormatter.ints().format(value));
+    NumberColumnFormatter ncf = NumberColumnFormatter.ints();
+    ncf.setColumnType(ColumnType.LONG);
+    assertEquals(Long.toString(value), ncf.format(value));
   }
 }
diff --git a/core/src/test/java/tech/tablesaw/io/csv/CsvWriterTest.java b/core/src/test/java/tech/tablesaw/io/csv/CsvWriterTest.java
index 0792e02c..d38785c3 100644
--- a/core/src/test/java/tech/tablesaw/io/csv/CsvWriterTest.java
+++ b/core/src/test/java/tech/tablesaw/io/csv/CsvWriterTest.java
@@ -70,6 +70,21 @@ public class CsvWriterTest {
         writer.toString().replaceAll("\\r\\n", "\n"));
   }
 
+  @Test
+  void printFormatter_double2() throws IOException {
+    Table table = Table.create("", DoubleColumn.create("percents"));
+    table
+        .doubleColumn("percents")
+        .setPrintFormatter(
+            new NumberColumnFormatter(NumberColumnFormatter.percent(2).getFormat(), "NA"));
+    table.doubleColumn("percents").append(Double.NaN).append(0.323).append(0.1192).append(1.0);
+    StringWriter writer = new StringWriter();
+    table.write().usingOptions(CsvWriteOptions.builder(writer).usePrintFormatters(true).build());
+    assertEquals(
+        "percents\n" + "NA\n" + "32.30%\n" + "11.92%\n" + "100.00%\n",
+        writer.toString().replaceAll("\\r\\n", "\n"));
+  }
+
   @Test
   void printFormatter_date() throws IOException {
     Table table = Table.create("", DateColumn.create("dates"));
@@ -99,6 +114,19 @@ public class CsvWriterTest {
         writer.toString().replaceAll("\\r\\n", "\n"));
   }
 
+  @Test
+  void printFormatter_int2() throws IOException {
+    Table table = Table.create("", IntColumn.create("ints"));
+    table.intColumn("ints").setPrintFormatter(new NumberColumnFormatter("NA"));
+    Integer missing = null;
+    table.intColumn("ints").append(102_123).append(2).append(missing).append(-1_232_132);
+    StringWriter writer = new StringWriter();
+    table.write().usingOptions(CsvWriteOptions.builder(writer).usePrintFormatters(true).build());
+    assertEquals(
+        "ints\n" + "102123\n" + "2\n" + "NA\n" + "-1232132\n",
+        writer.toString().replaceAll("\\r\\n", "\n"));
+  }
+
   @Test
   void printFormatter_float() throws IOException {
     Table table = Table.create("", FloatColumn.create("floats"));
@@ -111,6 +139,22 @@ public class CsvWriterTest {
         writer.toString().replaceAll("\\r\\n", "\n"));
   }
 
+  @Test
+  void printFormatter_float2() throws IOException {
+    Table table = Table.create("", FloatColumn.create("floats"));
+    table
+        .floatColumn("floats")
+        .setPrintFormatter(
+            new NumberColumnFormatter(
+                NumberColumnFormatter.fixedWithGrouping(2).getFormat(), "NA"));
+    table.floatColumn("floats").append(032.3f).append(0.1192f).appendObj(null).append(1001.0f);
+    StringWriter writer = new StringWriter();
+    table.write().usingOptions(CsvWriteOptions.builder(writer).usePrintFormatters(true).build());
+    assertEquals(
+        "floats\n" + "32.30\n" + "0.12\n" + "NA\n" + "\"1,001.00\"\n",
+        writer.toString().replaceAll("\\r\\n", "\n"));
+  }
+
   @Test
   void printFormatter_datetime() throws IOException {
     Table table = Table.create("", DateTimeColumn.create("dates"));