Repository: OpenRefine/OpenRefine (java)

Issue:
Exception error raised when opening a CSV with large number of columns
I have some CSVs and TSVs which have hundreds of columns and which don't open in OpenRefine. I don't really mind that there is a limit on columns, but when it attempts to open such files (into the parsing page after selecting the file) a popup window is shown with a java exception. Here is the message in the popup I got when opening a TSV with 1176 columns:

java.lang.ArrayIndexOutOfBoundsException - Index 512 out of bounds for length 512
Parser Configuration: TsvParserSettings:
	Auto configuration enabled=true
	Auto-closing enabled=true
	Column reordering enabled=true
	Header extraction enabled=null
	Headers=null
	Ignore leading whitespaces=true
	Ignore trailing whitespaces=true
	Input buffer size=1048576
	Input reading on separate thread=true
	Length of content displayed on error=-1
	Line separator detection enabled=false
	Maximum number of characters per column=262144
	Maximum number of columns=512
	Null value=null
	Number of records to read=all
	Processor=none
	Restricting data in exceptions=false
	RowProcessor error handler=null
	Selected fields=none
	Skip bits as whitespace=true
	Skip empty lines=trueFormat configuration:
	TsvFormat:
		Comment character=#
		Escape character=\
		Line separator (normalized)=\n
		Line separator sequence=\n
Internal state when error was thrown: line=0, column=513, record=0, charIndex=18558

I think OR should, instead, throw up a short message stating the column limit (I'm guessing it's 512 based on the above error). It would be even better if OR could handle more columns.

I'm using OpenRefine 3.8.2 on Safari 17.5 on MacOS 14.5


Pull request:
Increase TSV/CSV column limit to 32K (from 512). Fixes #6723
Fixes #6723 

Changes proposed in this pull request:
- Increase TSV/CSV column limit to 32K (from 512)
- Make cryptic test assertion failures easier to understand



Patch:
diff --git a/extensions/database/pom.xml b/extensions/database/pom.xml
index 1403e93f5..f79fa6797 100644
--- a/extensions/database/pom.xml
+++ b/extensions/database/pom.xml
@@ -104,7 +104,7 @@
     <dependency>
       <groupId>org.xerial</groupId>
       <artifactId>sqlite-jdbc</artifactId>
-      <version>3.46.0.0</version>
+      <version>3.46.0.1</version>
     </dependency>
     <dependency>
       <groupId>com.fasterxml.jackson.core</groupId>
diff --git a/extensions/gdata/pom.xml b/extensions/gdata/pom.xml
index 1dee209aa..3842e9f18 100644
--- a/extensions/gdata/pom.xml
+++ b/extensions/gdata/pom.xml
@@ -83,7 +83,7 @@
     <dependency>
       <groupId>com.google.apis</groupId>
       <artifactId>google-api-services-drive</artifactId>
-      <version>v3-rev20240628-2.0.0</version>
+      <version>v3-rev20240722-2.0.0</version>
     </dependency>
     <dependency>
       <groupId>com.google.apis</groupId>
diff --git a/main/src/com/google/refine/importers/SeparatorBasedImporter.java b/main/src/com/google/refine/importers/SeparatorBasedImporter.java
index 9cf332947..56a3614b3 100644
--- a/main/src/com/google/refine/importers/SeparatorBasedImporter.java
+++ b/main/src/com/google/refine/importers/SeparatorBasedImporter.java
@@ -69,6 +69,10 @@ import com.google.refine.util.JSONUtilities;
 
 public class SeparatorBasedImporter extends TabularImportingParserBase {
 
+    // Excel limits: 1M rows x 16K columns, 32K characters max per cell
+    public static final int MAX_COLUMNS = 16 * 1024; // default 512
+    // TODO: Perhaps use a lower default and make user configurable?
+    public static final int MAX_CHARACTERS_PER_CELL = 32 * 1024; // default 4096
     public static final int GUESSER_LINE_COUNT = 100;
     char DEFAULT_QUOTE_CHAR = new CsvParserSettings().getFormat().getQuote();
 
@@ -145,7 +149,8 @@ public class SeparatorBasedImporter extends TabularImportingParserBase {
         AbstractParser parser;
         if (tsv) {
             TsvParserSettings settings = new TsvParserSettings();
-            settings.setMaxCharsPerColumn(256 * 1024); // TODO: Perhaps use a lower default and make user configurable?
+            settings.setMaxCharsPerColumn(MAX_CHARACTERS_PER_CELL);
+            settings.setMaxColumns(MAX_COLUMNS);
             settings.setIgnoreLeadingWhitespaces(false);
             settings.setIgnoreTrailingWhitespaces(false);
             parser = new TsvParser(settings);
@@ -161,7 +166,8 @@ public class SeparatorBasedImporter extends TabularImportingParserBase {
                 settings.setUnescapedQuoteHandling(UnescapedQuoteHandling.RAISE_ERROR);
             }
             settings.setKeepQuotes(!processQuotes);
-            settings.setMaxCharsPerColumn(256 * 1024); // TODO: Perhaps use a lower default and make user configurable?
+            settings.setMaxCharsPerColumn(MAX_CHARACTERS_PER_CELL);
+            settings.setMaxColumns(MAX_COLUMNS);
             parser = new CsvParser(settings);
         }
         try (final LineNumberReader lnReader = new LineNumberReader(reader);) {
diff --git a/pom.xml b/pom.xml
index cb9da7068..d7d4d5ad3 100644
--- a/pom.xml
+++ b/pom.xml
@@ -98,7 +98,7 @@
     <jsoup.version>1.18.1</jsoup.version>
     <odfdom-java.version>0.12.0</odfdom-java.version> <!-- do not update to 0.10.0, see issue #4397 -->
     <vicino.version>1.2.1</vicino.version>
-    <clojure.version>1.11.3</clojure.version>
+    <clojure.version>1.11.4</clojure.version>
     <httpclient5.version>5.3.1</httpclient5.version>
     <httpcore.version>4.4.16</httpcore.version>
     <univocity-csv.version>2.9.1</univocity-csv.version>
@@ -109,7 +109,7 @@
     <juniversalchardet.version>2.5.0</juniversalchardet.version>
     <testng.version>7.10.2</testng.version>
     <httpmime.version>4.5.14</httpmime.version>
-    <owasp-encoder.version>1.2.3</owasp-encoder.version>
+    <owasp-encoder.version>1.3.0</owasp-encoder.version>
     <jasypt.version>1.9.3</jasypt.version>
     <mockito.version>5.12.0</mockito.version>
 


Test patch:
diff --git a/main/tests/cypress/package.json b/main/tests/cypress/package.json
index cb6f849df..5f0cb8503 100644
--- a/main/tests/cypress/package.json
+++ b/main/tests/cypress/package.json
@@ -16,7 +16,7 @@
         "lint": "prettier --check . && eslint ."
     },
     "dependencies": {
-        "cypress": "13.13.1",
+        "cypress": "13.13.2",
         "cypress-file-upload": "^5.0.8"
     },
     "devDependencies": {
diff --git a/main/tests/cypress/yarn.lock b/main/tests/cypress/yarn.lock
index 65866a6b7..f7ed19f18 100644
--- a/main/tests/cypress/yarn.lock
+++ b/main/tests/cypress/yarn.lock
@@ -7,10 +7,10 @@
   resolved "https://registry.yarnpkg.com/@aashutoshrathi/word-wrap/-/word-wrap-1.2.6.tgz#bd9154aec9983f77b3a034ecaa015c2e4201f6cf"
   integrity sha512-1Yjs2SvM8TflER/OD3cOjhWWOZb58A2t7wpE2S9XfBYTiIl+XFhQG2bjy4Pu1I+EAlCNUzRDYDdFwFYUKvXcIA==
 
-"@cypress/request@^3.0.0":
-  version "3.0.0"
-  resolved "https://registry.yarnpkg.com/@cypress/request/-/request-3.0.0.tgz#7f58dfda087615ed4e6aab1b25fffe7630d6dd85"
-  integrity sha512-GKFCqwZwMYmL3IBoNeR2MM1SnxRIGERsQOTWeQKoYBt2JLqcqiy7JXqO894FLrpjZYqGxW92MNwRH2BN56obdQ==
+"@cypress/request@^3.0.1":
+  version "3.0.1"
+  resolved "https://registry.yarnpkg.com/@cypress/request/-/request-3.0.1.tgz#72d7d5425236a2413bd3d8bb66d02d9dc3168960"
+  integrity sha512-TWivJlJi8ZDx2wGOw1dbLuHJKUYX7bWySw377nlnGOW3hP9/MUKIsEdXT/YngWxVdgNCHRBmFlBipE+5/2ZZlQ==
   dependencies:
     aws-sign2 "~0.7.0"
     aws4 "^1.8.0"
@@ -25,7 +25,7 @@
     json-stringify-safe "~5.0.1"
     mime-types "~2.1.19"
     performance-now "^2.1.0"
-    qs "~6.10.3"
+    qs "6.10.4"
     safe-buffer "^5.1.2"
     tough-cookie "^4.1.3"
     tunnel-agent "^0.6.0"
@@ -436,12 +436,12 @@ cypress-file-upload@^5.0.8:
   resolved "https://registry.yarnpkg.com/cypress-file-upload/-/cypress-file-upload-5.0.8.tgz#d8824cbeaab798e44be8009769f9a6c9daa1b4a1"
   integrity sha512-+8VzNabRk3zG6x8f8BWArF/xA/W0VK4IZNx3MV0jFWrJS/qKn8eHfa5nU73P9fOQAgwHFJx7zjg4lwOnljMO8g==
 
-cypress@13.13.1:
-  version "13.13.1"
-  resolved "https://registry.yarnpkg.com/cypress/-/cypress-13.13.1.tgz#860c1142a6e58ea412a764f0ce6ad07567721129"
-  integrity sha512-8F9UjL5MDUdgC/S5hr8CGLHbS5gGht5UOV184qc2pFny43fnkoaKxlzH/U6//zmGu/xRTaKimNfjknLT8+UDFg==
+cypress@13.13.2:
+  version "13.13.2"
+  resolved "https://registry.yarnpkg.com/cypress/-/cypress-13.13.2.tgz#c71f8d92056c430b1b879e5313f6de25ccce0eda"
+  integrity sha512-PvJQU33933NvS1StfzEb8/mu2kMy4dABwCF+yd5Bi7Qly1HOVf+Bufrygee/tlmty/6j5lX+KIi8j9Q3JUMbhA==
   dependencies:
-    "@cypress/request" "^3.0.0"
+    "@cypress/request" "^3.0.1"
     "@cypress/xvfb" "^1.2.4"
     "@types/sinonjs__fake-timers" "8.1.1"
     "@types/sizzle" "^2.3.2"
@@ -1317,10 +1317,10 @@ punycode@^2.1.0, punycode@^2.1.1:
   resolved "https://registry.yarnpkg.com/punycode/-/punycode-2.1.1.tgz"
   integrity sha512-XRsRjdf+j5ml+y/6GKHPZbrF/8p2Yga0JPtdqTIY2Xe5ohJPD9saDJJLPvp9+NSBprVvevdXZybnj2cv8OEd0A==
 
-qs@~6.10.3:
-  version "6.10.5"
-  resolved "https://registry.yarnpkg.com/qs/-/qs-6.10.5.tgz#974715920a80ff6a262264acd2c7e6c2a53282b4"
-  integrity sha512-O5RlPh0VFtR78y79rgcgKK4wbAI0C5zGVLztOIdpWX6ep368q5Hv6XRxDvXuZ9q3C6v+e3n8UfZZJw7IIG27eQ==
+qs@6.10.4:
+  version "6.10.4"
+  resolved "https://registry.yarnpkg.com/qs/-/qs-6.10.4.tgz#6a3003755add91c0ec9eacdc5f878b034e73f9e7"
+  integrity sha512-OQiU+C+Ds5qiH91qh/mg0w+8nwQuLjM4F4M/PbmhDOoYehPh+Fb0bDjtR1sOvy7YKxvj28Y/M0PhP5uVX0kB+g==
   dependencies:
     side-channel "^1.0.4"
 
diff --git a/main/tests/server/src/com/google/refine/importers/ImporterTest.java b/main/tests/server/src/com/google/refine/importers/ImporterTest.java
index ec6649714..75e5cdac4 100644
--- a/main/tests/server/src/com/google/refine/importers/ImporterTest.java
+++ b/main/tests/server/src/com/google/refine/importers/ImporterTest.java
@@ -96,7 +96,7 @@ public abstract class ImporterTest extends RefineTest {
                 -1,
                 options,
                 exceptions);
-        assertEquals(exceptions.size(), 0);
+        assertEquals(exceptions.size(), 0, "Unexpected exception(s) thrown: " + exceptions);
         project.update();
     }
 
@@ -111,7 +111,7 @@ public abstract class ImporterTest extends RefineTest {
                 -1,
                 options,
                 exceptions);
-        assertEquals(exceptions.size(), 0);
+        assertEquals(exceptions.size(), 0, "Unexpected exception(s) thrown: " + exceptions);
         project.update();
     }
 
@@ -143,7 +143,7 @@ public abstract class ImporterTest extends RefineTest {
                 -1,
                 options,
                 exceptions);
-        assertEquals(exceptions.size(), 0);
+        assertEquals(exceptions.size(), 0, "Unexpected exception(s) thrown: " + exceptions);
         XmlImportUtilities.createColumnsFromImport(project, rootColumnGroup);
         project.columnModel.update();
     }
diff --git a/main/tests/server/src/com/google/refine/importers/SeparatorBasedImporterTests.java b/main/tests/server/src/com/google/refine/importers/SeparatorBasedImporterTests.java
index 92357de95..4f59715c0 100644
--- a/main/tests/server/src/com/google/refine/importers/SeparatorBasedImporterTests.java
+++ b/main/tests/server/src/com/google/refine/importers/SeparatorBasedImporterTests.java
@@ -46,6 +46,7 @@ import java.util.List;
 
 import com.fasterxml.jackson.databind.node.ObjectNode;
 import org.apache.commons.io.FileUtils;
+import org.apache.commons.io.input.CharSequenceReader;
 import org.slf4j.LoggerFactory;
 import org.testng.Assert;
 import org.testng.annotations.AfterMethod;
@@ -298,6 +299,39 @@ public class SeparatorBasedImporterTests extends ImporterTest {
         assertProjectEquals(project, expectedProject);
     }
 
+    @Test(dataProvider = "CSV-TSV-AutoDetermine")
+    public void readManyColumns(String sep) {
+        // create input
+        int width = 16 * 1024; // Excel supports 16K columns max, so let's test that
+        String inputSeparator = sep == null ? "\t" : sep;
+        StringBuilder input = new StringBuilder();
+        String[] colNames = new String[width];
+        for (int i = 0; i < width; i++) {
+            String name = "col" + (i + 1);
+            input.append(name + inputSeparator);
+            colNames[i] = name;
+        }
+        input = input.deleteCharAt(input.length() - 1); // we don't need the last separator
+        input.append('\n');
+        String[] data = new String[width];
+        for (int i = 0; i < width; i++) {
+            String value = "data" + (i + 1);
+            input.append(value + inputSeparator);
+            data[i] = value;
+        }
+        input = input.deleteCharAt(input.length() - 1); // we don't need the last separator
+
+        prepareOptions(sep, -1, 0, 0, 1, false, false);
+        parseOneFile(SUT, new CharSequenceReader(input));
+
+        Project expectedProject = createProject(
+                colNames,
+                new Serializable[][] {
+                        data,
+                });
+        assertProjectEquals(project, expectedProject);
+    }
+
     @Test(groups = {}, dataProvider = "CSV-TSV-AutoDetermine")
     public void readQuotedData(String sep) {
         // create input