Repository: jtablesaw/tablesaw
Pull request #875 (2021-01-25 21:32:18)

Fix for second Issue in #871 (ColumnType detection and text columns)
Thanks for contributing.

- [x] Tick to sign-off your agreement to the [Developer Certificate of Origin (DCO) 1.1](https://developercertificate.org)

## Description

What was changed

## Testing

Did you add a unit test?
yes



Patch:
diff --git a/core/src/main/java/tech/tablesaw/io/ColumnTypeDetector.java b/core/src/main/java/tech/tablesaw/io/ColumnTypeDetector.java
index bf20be51..5eef7383 100644
--- a/core/src/main/java/tech/tablesaw/io/ColumnTypeDetector.java
+++ b/core/src/main/java/tech/tablesaw/io/ColumnTypeDetector.java
@@ -1,5 +1,6 @@
 package tech.tablesaw.io;
 
+import static tech.tablesaw.api.ColumnType.STRING;
 import static tech.tablesaw.api.ColumnType.TEXT;
 
 import java.util.ArrayList;
@@ -9,7 +10,6 @@ import java.util.List;
 import java.util.concurrent.CopyOnWriteArrayList;
 import tech.tablesaw.api.ColumnType;
 import tech.tablesaw.columns.AbstractColumnParser;
-import tech.tablesaw.columns.strings.StringColumnType;
 
 public class ColumnTypeDetector {
 
@@ -98,8 +98,9 @@ public class ColumnTypeDetector {
     // now detect
     for (List<String> valuesList : columnData) {
       ColumnType detectedType = detectType(valuesList, options);
-      if (detectedType.equals(StringColumnType.STRING)
-          && rowCount > STRING_COLUMN_ROW_COUNT_CUTOFF) {
+      if (detectedType.equals(STRING)
+          && rowCount > STRING_COLUMN_ROW_COUNT_CUTOFF
+          && options.columnTypesToDetect().contains(TEXT)) {
         HashSet<String> unique = new HashSet<>(valuesList);
         double uniquePct = unique.size() / (valuesList.size() * 1.0);
         if (uniquePct > STRING_COLUMN_CUTOFF) {
diff --git a/core/src/main/java/tech/tablesaw/io/ReadOptions.java b/core/src/main/java/tech/tablesaw/io/ReadOptions.java
index 9f5d50fb..3c877e40 100644
--- a/core/src/main/java/tech/tablesaw/io/ReadOptions.java
+++ b/core/src/main/java/tech/tablesaw/io/ReadOptions.java
@@ -44,7 +44,7 @@ public class ReadOptions {
 
   private static final List<ColumnType> DEFAULT_TYPES =
       Lists.newArrayList(
-          LOCAL_DATE_TIME, LOCAL_TIME, LOCAL_DATE, BOOLEAN, INTEGER, LONG, DOUBLE, STRING);
+          LOCAL_DATE_TIME, LOCAL_TIME, LOCAL_DATE, BOOLEAN, INTEGER, LONG, DOUBLE, STRING, TEXT);
 
   /**
    * An extended list of types that are used if minimizeColumnSizes is true. By including extra
@@ -52,7 +52,7 @@ public class ReadOptions {
    * for the programmer if, for example, they will subsequently modify the data in a way that
    * exceeds the range of the type.
    */
-  private static final List<ColumnType> EXTENDED_TYPES =
+  protected static final List<ColumnType> EXTENDED_TYPES =
       Lists.newArrayList(
           LOCAL_DATE_TIME,
           LOCAL_TIME,
diff --git a/core/src/main/java/tech/tablesaw/io/csv/CsvReadOptions.java b/core/src/main/java/tech/tablesaw/io/csv/CsvReadOptions.java
index 7c0a37fa..4088eea0 100644
--- a/core/src/main/java/tech/tablesaw/io/csv/CsvReadOptions.java
+++ b/core/src/main/java/tech/tablesaw/io/csv/CsvReadOptions.java
@@ -21,6 +21,8 @@ import java.io.Reader;
 import java.io.StringReader;
 import java.net.URL;
 import java.time.format.DateTimeFormatter;
+import java.util.ArrayList;
+import java.util.List;
 import java.util.Locale;
 import tech.tablesaw.api.ColumnType;
 import tech.tablesaw.io.ReadOptions;
@@ -252,6 +254,20 @@ public class CsvReadOptions extends ReadOptions {
       return this;
     }
 
+    @Override
+    public Builder columnTypesToDetect(List<ColumnType> columnTypesToDetect) {
+      // Types need to be in certain order as more general types like string come last
+      // Otherwise everything will be parsed as a string
+      List<ColumnType> orderedTypes = new ArrayList<>();
+      for (ColumnType t : EXTENDED_TYPES) {
+        if (columnTypesToDetect.contains(t)) {
+          orderedTypes.add(t);
+        }
+      }
+      this.columnTypesToDetect = orderedTypes;
+      return this;
+    }
+
     @Override
     public Builder tableName(String tableName) {
       super.tableName(tableName);


Test patch:
diff --git a/core/src/test/java/tech/tablesaw/io/csv/CsvReadPerformanceTest.java b/core/src/test/java/tech/tablesaw/io/csv/CsvReadPerformanceTest.java
deleted file mode 100644
index f11ff0f0..00000000
--- a/core/src/test/java/tech/tablesaw/io/csv/CsvReadPerformanceTest.java
+++ /dev/null
@@ -1,90 +0,0 @@
-/*
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package tech.tablesaw.io.csv;
-
-import static tech.tablesaw.api.ColumnType.INTEGER;
-import static tech.tablesaw.api.ColumnType.SHORT;
-import static tech.tablesaw.api.ColumnType.STRING;
-import static tech.tablesaw.api.ColumnType.TEXT;
-
-import com.google.common.base.Stopwatch;
-import java.util.concurrent.TimeUnit;
-import tech.tablesaw.api.ColumnType;
-import tech.tablesaw.api.Table;
-
-public class CsvReadPerformanceTest {
-
-  private static final ColumnType[] types = {
-    TEXT, // 0     ID
-    STRING, // 1     CNTYFIPS
-    TEXT, // 2     Ori
-    STRING, // 3     State
-    TEXT, // 4     Agency
-    STRING, // 5     Agentype
-    STRING, // 6     Source
-    STRING, // 7     Solved
-    SHORT, // 8     Year
-    STRING, // 9     StateName
-    STRING, // 10    Month
-    SHORT, // 11    Incident
-    STRING, // 12    ActionType
-    STRING, // 13    Homicide
-    STRING, // 14    Situation
-    SHORT, // 15    VicAge
-    STRING, // 16    VicSex
-    STRING, // 17    VicRace
-    STRING, // 18    VicEthnic
-    SHORT, // 19    OffAge
-    STRING, // 20    OffSex
-    STRING, // 21    OffRace
-    STRING, // 22    OffEthnic
-    STRING, // 23    Weapon
-    STRING, // 24    Relationship
-    STRING, // 25    Circumstance
-    STRING, // 26    Subcircum
-    SHORT, // 27    VicCount
-    SHORT, // 28    OffCount
-    INTEGER, // 29    FileDate
-    STRING, // 30    fstate
-    STRING, // 31    MSA
-  };
-
-  /** Usage example using a Tornado data set */
-  public static void main(String[] args) throws Exception {
-
-    Stopwatch stopwatch = Stopwatch.createStarted();
-    Table details = Table.read().csv("../data/SHR76_16.csv");
-    stopwatch.stop();
-    System.out.println(
-        "Large file (752,313 rows) read: "
-            + stopwatch.elapsed(TimeUnit.MILLISECONDS)
-            + " ms with type detection.");
-    System.out.println(details.shape());
-    System.out.println(details.structure().printAll());
-    System.out.println(details);
-
-    stopwatch.reset();
-    stopwatch.start();
-    details =
-        Table.read().csv(CsvReadOptions.builder("../data/SHR76_16.csv").columnTypes(types).build());
-    stopwatch.stop();
-
-    System.out.println(
-        "Large file (752,313 rows) read: "
-            + stopwatch.elapsed(TimeUnit.MILLISECONDS)
-            + " ms without type detection.");
-    System.out.println(details.shape());
-  }
-}
diff --git a/core/src/test/java/tech/tablesaw/io/csv/CsvReaderTest.java b/core/src/test/java/tech/tablesaw/io/csv/CsvReaderTest.java
index f9792653..4b0e4240 100644
--- a/core/src/test/java/tech/tablesaw/io/csv/CsvReaderTest.java
+++ b/core/src/test/java/tech/tablesaw/io/csv/CsvReaderTest.java
@@ -20,15 +20,7 @@ import static org.junit.jupiter.api.Assertions.assertEquals;
 import static org.junit.jupiter.api.Assertions.assertFalse;
 import static org.junit.jupiter.api.Assertions.assertNotNull;
 import static org.junit.jupiter.api.Assertions.assertThrows;
-import static tech.tablesaw.api.ColumnType.DOUBLE;
-import static tech.tablesaw.api.ColumnType.FLOAT;
-import static tech.tablesaw.api.ColumnType.INTEGER;
-import static tech.tablesaw.api.ColumnType.LOCAL_DATE;
-import static tech.tablesaw.api.ColumnType.LOCAL_DATE_TIME;
-import static tech.tablesaw.api.ColumnType.LOCAL_TIME;
-import static tech.tablesaw.api.ColumnType.SHORT;
-import static tech.tablesaw.api.ColumnType.SKIP;
-import static tech.tablesaw.api.ColumnType.STRING;
+import static tech.tablesaw.api.ColumnType.*;
 
 import com.univocity.parsers.common.TextParsingException;
 import java.io.File;
@@ -110,6 +102,23 @@ public class CsvReaderTest {
     table.removeColumns("stop_desc");
   }
 
+  @Test
+  void textColumnShutoff() throws IOException {
+
+    Table table = Table.read().csv(CsvReadOptions.builder("../data/cities-states-zipcode.csv"));
+    ColumnType[] types = {INTEGER, STRING, DOUBLE, BOOLEAN};
+    List<ColumnType> typesToDetect = Arrays.asList(types);
+    Table table2 =
+        Table.read()
+            .csv(
+                CsvReadOptions.builder("../data/cities-states-zipcode.csv")
+                    .columnTypesToDetect(typesToDetect)
+                    .build());
+
+    assertEquals(table.column("WorldRegion").type(), TEXT);
+    assertEquals(table2.column("WorldRegion").type(), STRING);
+  }
+
   @Test
   public void testWithColumnSKIP() throws IOException {
     // Read the CSV file