Repository: jtablesaw/tablesaw
Pull request #867 (2021-01-20 17:34:35)

implemented optional renaming of duplicate columns
Thanks for contributing.

- [x] Tick to sign-off your agreement to the [Developer Certificate of Origin (DCO) 1.1](https://developercertificate.org)

## Description

Allows the user to specify that a file containing duplicate column names should be imported, with the duplicate names modified. Current behavior is to throw an exception, forcing the user to modify the file externally. 

## Testing

Did you add a unit test?
yes



Patch:
diff --git a/core/src/main/java/tech/tablesaw/io/FileReader.java b/core/src/main/java/tech/tablesaw/io/FileReader.java
index a8be58a7..d84ee956 100644
--- a/core/src/main/java/tech/tablesaw/io/FileReader.java
+++ b/core/src/main/java/tech/tablesaw/io/FileReader.java
@@ -70,8 +70,11 @@ public abstract class FileReader {
   /** Returns the column names for each column in the source. */
   public String[] getColumnNames(
       ReadOptions options, ColumnType[] types, AbstractParser<?> parser) {
+
     if (options.header()) {
+
       String[] headerNames = parser.parseNext();
+
       // work around issue where Univocity returns null if a column has no header.
       for (int i = 0; i < headerNames.length; i++) {
         if (headerNames[i] == null) {
@@ -80,6 +83,9 @@ public abstract class FileReader {
           headerNames[i] = headerNames[i].trim();
         }
       }
+      if (options.allowDuplicateColumnNames()) {
+        renameDuplicateColumnHeaders(headerNames);
+      }
       return headerNames;
     } else {
       // Placeholder column names for when the file read has no header
@@ -91,6 +97,28 @@ public abstract class FileReader {
     }
   }
 
+  /**
+   * Renames any column header that appears more than once. Subsequent appearances have "-[count]"
+   * appended; For example, the first (or only) appearance of "foo" is named "foo", the second
+   * appearance is named "foo-2" The header array is modified in place.
+   *
+   * @param headerNames The header names to be potentially adjusted.
+   */
+  private void renameDuplicateColumnHeaders(String[] headerNames) {
+    Map<String, Integer> nameCounter = new HashMap<>();
+    for (int i = 0; i < headerNames.length; i++) {
+      String name = headerNames[i];
+      Integer count = nameCounter.get(name);
+      if (count == null) {
+        nameCounter.put(name, 1);
+      } else {
+        count++;
+        nameCounter.put(name, count);
+        headerNames[i] = name + "-" + count;
+      }
+    }
+  }
+
   protected Table parseRows(
       ReadOptions options,
       boolean headerOnly,
diff --git a/core/src/main/java/tech/tablesaw/io/ReadOptions.java b/core/src/main/java/tech/tablesaw/io/ReadOptions.java
index 28332412..9f5d50fb 100644
--- a/core/src/main/java/tech/tablesaw/io/ReadOptions.java
+++ b/core/src/main/java/tech/tablesaw/io/ReadOptions.java
@@ -78,6 +78,7 @@ public class ReadOptions {
   protected final boolean minimizeColumnSizes;
   protected final int maxCharsPerColumn;
   protected final boolean ignoreZeroDecimal;
+  protected final boolean allowDuplicateColumnNames;
 
   protected final DateTimeFormatter dateFormatter;
   protected final DateTimeFormatter dateTimeFormatter;
@@ -103,6 +104,8 @@ public class ReadOptions {
     timeFormatter = builder.timeFormatter;
     dateTimeFormatter = builder.dateTimeFormatter;
 
+    allowDuplicateColumnNames = builder.allowDuplicateColumnNames;
+
     if (builder.locale == null) {
       locale = Locale.getDefault();
     } else {
@@ -118,6 +121,10 @@ public class ReadOptions {
     return tableName;
   }
 
+  public boolean allowDuplicateColumnNames() {
+    return allowDuplicateColumnNames;
+  }
+
   public List<ColumnType> columnTypesToDetect() {
     return columnTypesToDetect;
   }
@@ -195,6 +202,7 @@ public class ReadOptions {
     protected boolean header = true;
     protected int maxCharsPerColumn = 4096;
     protected boolean ignoreZeroDecimal = DEFAULT_IGNORE_ZERO_DECIMAL;
+    private boolean allowDuplicateColumnNames = false;
 
     protected Builder() {
       source = null;
@@ -244,6 +252,11 @@ public class ReadOptions {
       return this;
     }
 
+    public Builder allowDuplicateColumnNames(Boolean allow) {
+      this.allowDuplicateColumnNames = allow;
+      return this;
+    }
+
     /** Deprecated. Use timeFormat(DateTimeFormatter dateFormat) instead */
     @Deprecated
     public Builder timeFormat(String timeFormat) {
diff --git a/core/src/main/java/tech/tablesaw/io/csv/CsvReadOptions.java b/core/src/main/java/tech/tablesaw/io/csv/CsvReadOptions.java
index 1b67420b..7c0a37fa 100644
--- a/core/src/main/java/tech/tablesaw/io/csv/CsvReadOptions.java
+++ b/core/src/main/java/tech/tablesaw/io/csv/CsvReadOptions.java
@@ -240,6 +240,18 @@ public class CsvReadOptions extends ReadOptions {
       return this;
     }
 
+    /**
+     * Enable reading of a table with duplicate column names. After the first appearance of a column
+     * name, subsequent appearances will have a number appended.
+     *
+     * @param allow if true, duplicate names will be allowed
+     */
+    @Override
+    public Builder allowDuplicateColumnNames(Boolean allow) {
+      super.allowDuplicateColumnNames(allow);
+      return this;
+    }
+
     @Override
     public Builder tableName(String tableName) {
       super.tableName(tableName);


Test patch:
diff --git a/core/src/test/java/tech/tablesaw/io/csv/CsvReaderTest.java b/core/src/test/java/tech/tablesaw/io/csv/CsvReaderTest.java
index 537eca1c..f9792653 100644
--- a/core/src/test/java/tech/tablesaw/io/csv/CsvReaderTest.java
+++ b/core/src/test/java/tech/tablesaw/io/csv/CsvReaderTest.java
@@ -64,6 +64,7 @@ import tech.tablesaw.io.AddCellToColumnException;
 public class CsvReaderTest {
 
   private static final String LINE_END = System.lineSeparator();
+  private static final String COMMA = ",";
 
   private final ColumnType[] bus_types = {SHORT, STRING, STRING, FLOAT, FLOAT};
   private final ColumnType[] bus_types_with_SKIP = {SHORT, STRING, SKIP, DOUBLE, DOUBLE};
@@ -123,6 +124,24 @@ public class CsvReaderTest {
     assertEquals("[stop_id, stop_name, stop_lat, stop_lon]", table.columnNames().toString());
   }
 
+  @Test
+  void allowDuplicateColumnNames() throws IOException {
+    final Reader reader1 =
+        new StringReader(
+            "Col1" + COMMA + "Col2" + LINE_END + "\"first\"" + COMMA + "second" + LINE_END);
+    Table noDupes = Table.read().csv(reader1);
+    assertEquals("Col1", noDupes.columnNames().get(0));
+    assertEquals("Col2", noDupes.columnNames().get(1));
+
+    final Reader reader2 =
+        new StringReader(
+            "Col1" + COMMA + "Col1" + LINE_END + "\"first\"" + COMMA + "second" + LINE_END);
+    Table dupes =
+        Table.read().csv(CsvReadOptions.builder(reader2).allowDuplicateColumnNames(true).build());
+    assertEquals("Col1", dupes.columnNames().get(0));
+    assertEquals("Col1-2", dupes.columnNames().get(1));
+  }
+
   @Test
   public void testWithColumnSKIPWithoutHeader() throws IOException {
     // Read the CSV file