Repository: jtablesaw/tablesaw
Pull request #483 (2019-03-07 18:30:11)

Support comment character in CSV files
Thanks for contributing.

- [X] Tick to sign-off your agreement to the [Developer Certificate of Origin (DCO) 1.1](https://developercertificate.org)

## Description

I've added the ability to ignore commented lines starting with a specified prefix character.

This is a simple change that just exposes functionality already in CsvFormat. 

## Testing

I've tested comments before the header and throughout the CSV file to make sure the shape of the file is correct and the commented out lines are ignored and don't affect any existing functionality.



Patch:
diff --git a/core/src/main/java/tech/tablesaw/io/csv/CsvReadOptions.java b/core/src/main/java/tech/tablesaw/io/csv/CsvReadOptions.java
index dd195132..3b02687f 100644
--- a/core/src/main/java/tech/tablesaw/io/csv/CsvReadOptions.java
+++ b/core/src/main/java/tech/tablesaw/io/csv/CsvReadOptions.java
@@ -30,6 +30,7 @@ public class CsvReadOptions extends ReadOptions {
     private final Character separator;
     private final String lineEnding;
     private final Integer maxNumberOfColumns;
+    private final Character commentPrefix;
 
     private CsvReadOptions(CsvReadOptions.Builder builder) {
 	super(builder);
@@ -38,6 +39,7 @@ public class CsvReadOptions extends ReadOptions {
         separator = builder.separator;
         lineEnding = builder.lineEnding;
         maxNumberOfColumns = builder.maxNumberOfColumns;
+        commentPrefix = builder.commentPrefix;
     }
 
     public static Builder builder(File file) {
@@ -139,13 +141,18 @@ public class CsvReadOptions extends ReadOptions {
         return maxNumberOfColumns;
     }
 
+    public Character commentPrefix() {
+        return commentPrefix;
+    }
+
     public static class Builder extends ReadOptions.Builder {
 
         private Character separator = ',';
         private String lineEnding;
         private ColumnType[] columnTypes;
         private Integer maxNumberOfColumns = 10_000;
-
+        private Character commentPrefix;
+        
         public Builder(File file) {
             super(file);
         }
@@ -244,6 +251,11 @@ public class CsvReadOptions extends ReadOptions {
             return this;
         }
 
+        public Builder commentPrefix(char commentPrefix) {
+            this.commentPrefix = commentPrefix;
+            return this;
+        }
+
         public CsvReadOptions build() {
             return new CsvReadOptions(this);
         }
diff --git a/core/src/main/java/tech/tablesaw/io/csv/CsvReader.java b/core/src/main/java/tech/tablesaw/io/csv/CsvReader.java
index ae1b0e16..5063febd 100644
--- a/core/src/main/java/tech/tablesaw/io/csv/CsvReader.java
+++ b/core/src/main/java/tech/tablesaw/io/csv/CsvReader.java
@@ -19,13 +19,15 @@ import com.univocity.parsers.common.AbstractParser;
 import com.univocity.parsers.csv.CsvFormat;
 import com.univocity.parsers.csv.CsvParser;
 import com.univocity.parsers.csv.CsvParserSettings;
-import org.apache.commons.lang3.tuple.Pair;
 import tech.tablesaw.api.ColumnType;
 import tech.tablesaw.api.Table;
 import tech.tablesaw.io.FileReader;
 import tech.tablesaw.io.TableBuildingUtils;
 
 import javax.annotation.concurrent.Immutable;
+
+import org.apache.commons.math3.util.Pair;
+
 import java.io.IOException;
 import java.io.Reader;
 import java.util.List;
@@ -67,7 +69,7 @@ public class CsvReader extends FileReader {
             types = detectColumnTypes(reader, options);
         }
 
-        return Pair.of(TableBuildingUtils.createReader(options, bytesCache), types);
+        return Pair.create(TableBuildingUtils.createReader(options, bytesCache), types);
     }
 
     public Table read(CsvReadOptions options) throws IOException {
@@ -76,8 +78,8 @@ public class CsvReader extends FileReader {
 
     private Table read(CsvReadOptions options, boolean headerOnly) throws IOException {
         Pair<Reader, ColumnType[]> pair = getReaderAndColumnTypes(options);
-        Reader reader = pair.getLeft();
-        ColumnType[] types = pair.getRight();
+        Reader reader = pair.getKey();
+        ColumnType[] types = pair.getValue();
 
         AbstractParser<?> parser = csvParser(options);
 
@@ -161,6 +163,9 @@ public class CsvReader extends FileReader {
         if (options.lineEnding() != null) {
             format.setLineSeparator(options.lineEnding());
         }
+        if(options.commentPrefix() != null) {
+            format.setComment(options.commentPrefix());
+        }
         return format;
     }
 
diff --git a/core/src/main/java/tech/tablesaw/io/fixed/FixedWidthReader.java b/core/src/main/java/tech/tablesaw/io/fixed/FixedWidthReader.java
index deedd606..59d2c66a 100644
--- a/core/src/main/java/tech/tablesaw/io/fixed/FixedWidthReader.java
+++ b/core/src/main/java/tech/tablesaw/io/fixed/FixedWidthReader.java
@@ -19,13 +19,15 @@ import com.univocity.parsers.common.AbstractParser;
 import com.univocity.parsers.fixed.FixedWidthFormat;
 import com.univocity.parsers.fixed.FixedWidthParser;
 import com.univocity.parsers.fixed.FixedWidthParserSettings;
-import org.apache.commons.lang3.tuple.Pair;
 import tech.tablesaw.api.ColumnType;
 import tech.tablesaw.api.Table;
 import tech.tablesaw.io.FileReader;
 import tech.tablesaw.io.TableBuildingUtils;
 
 import javax.annotation.concurrent.Immutable;
+
+import org.apache.commons.math3.util.Pair;
+
 import java.io.IOException;
 import java.io.Reader;
 import java.util.List;
@@ -67,7 +69,7 @@ public class FixedWidthReader extends FileReader {
             types = detectColumnTypes(reader, options);
         }
 
-        return Pair.of(TableBuildingUtils.createReader(options, bytesCache), types);
+        return Pair.create(TableBuildingUtils.createReader(options, bytesCache), types);
     }
 
     public Table read(FixedWidthReadOptions options) throws IOException {
@@ -76,8 +78,8 @@ public class FixedWidthReader extends FileReader {
 
     private Table read(FixedWidthReadOptions options, boolean headerOnly) throws IOException {
         Pair<Reader, ColumnType[]> pair = getReaderAndColumnTypes(options);
-        Reader reader = pair.getLeft();
-        ColumnType[] types = pair.getRight();
+        Reader reader = pair.getKey();
+        ColumnType[] types = pair.getValue();
 
         FixedWidthParser parser = fixedWidthParser(options);
 
diff --git a/data/with_comments.csv b/data/with_comments.csv
new file mode 100644
index 00000000..45330b85
--- /dev/null
+++ b/data/with_comments.csv
@@ -0,0 +1,11 @@
+#comment1
+#comment2
+Products,Sales,Market_Share
+# comment3
+a,5500,3
+# comment,with,some,rows
+#comment5
+b,12200,4
+c,60000,33
+#comment,with,rows
+#footer comment
\ No newline at end of file


Test patch:
diff --git a/core/src/test/java/tech/tablesaw/io/csv/CsvReaderTest.java b/core/src/test/java/tech/tablesaw/io/csv/CsvReaderTest.java
index b2bc9a78..3bb6b915 100644
--- a/core/src/test/java/tech/tablesaw/io/csv/CsvReaderTest.java
+++ b/core/src/test/java/tech/tablesaw/io/csv/CsvReaderTest.java
@@ -508,4 +508,13 @@ public class CsvReaderTest {
         assertEquals("1 rows X 10001 cols", table1.shape());
     }
 
+    @Test
+    public void testSkipLinesWithComments() throws Exception {
+        Table table1 = Table.read().csv(CsvReadOptions
+                .builder("../data/with_comments.csv")
+                .maxNumberOfColumns(3)
+                .commentPrefix('#')
+                .header(true));
+        assertEquals("3 rows X 3 cols", table1.shape());
+    }
 }