Repository: OpenRefine/OpenRefine (java)

Issue:
File type detection fails when multiple files are imported at once
The heuristic to detect file types seems to have a weird behavior when working on multiple files, even though those files have a consistent file type which OpenRefine is able to detect correctly when they are imported individually

### To Reproduce
Steps to reproduce the behavior:

Create two text files `test.md` and `test2.md`, both containing the following content:
```
## Markdown test

This is a [test](https://gnu.org) test
```

2. Import both of those files at the same time (by selecting them both in the same folder)

### Current Results

The files are incorrectly detected as Excel files, which obviously fails.

<!-- What results occurred or were shown. -->

### Expected Behavior

They should be detected as text files (which indeed happens when selecting only one file)

### Screenshots

![image](https://github.com/user-attachments/assets/7b44966a-355e-48bc-abe7-008a24380ac0)


### Versions<!--  (please complete the following information)-->
 - Operating System: Debian testing
 - JRE or JDK Version: 19
 - OpenRefine: 3.9-SNAPSHOT
 
### Additional context
Observed at Wikimania 2024


Pull request:
File type detection fails when multiple files are imported at once
File format info was not getting set when multiple files are uploaded.

Fixes #6772 

Changes proposed in this pull request:
- Set format in the import flow for multiple files
- Identify format based on the file, added guessBetterFormat() method
- Tested with multiple file types the format is getting set correctly 


Patch:
diff --git a/main/src/com/google/refine/importers/LineBasedFormatGuesser.java b/main/src/com/google/refine/importers/LineBasedFormatGuesser.java
index cff54e771..8987b8b71 100644
--- a/main/src/com/google/refine/importers/LineBasedFormatGuesser.java
+++ b/main/src/com/google/refine/importers/LineBasedFormatGuesser.java
@@ -35,7 +35,7 @@ public class LineBasedFormatGuesser implements FormatGuesser {
 
     @Override
     public String guess(File file, String encoding, String seedFormat) {
-        SeparatorBasedImporter.Separator sep = SeparatorBasedImporter.guessSeparator(file, encoding);
+        SeparatorBasedImporter.Separator sep = SeparatorBasedImporter.guessSeparator(file, encoding, true);
         if (sep != null) {
             return "text/line-based/*sv";
         }
diff --git a/main/src/com/google/refine/importing/ImportingUtilities.java b/main/src/com/google/refine/importing/ImportingUtilities.java
index 6209f9a7b..0e4898964 100644
--- a/main/src/com/google/refine/importing/ImportingUtilities.java
+++ b/main/src/com/google/refine/importing/ImportingUtilities.java
@@ -439,6 +439,7 @@ public class ImportingUtilities {
                             calculateProgressPercent(update.totalExpectedSize, update.totalRetrievedSize));
 
                     JSONUtilities.safePut(fileRecord, "size", saveStreamToFile(stream, file, null));
+                    JSONUtilities.safePut(fileRecord, "format", guessBetterFormat(file, request.getCharacterEncoding(), "text"));
                     // TODO: This needs to be refactored to be able to test import from archives
                     if (postProcessRetrievedFile(rawDataDir, file, fileRecord, fileRecords, progress)) {
                         archiveCount++;
@@ -1015,27 +1016,34 @@ public class ImportingUtilities {
 
             if (location != null) {
                 File file = new File(job.getRawDataDir(), location);
+                bestFormat = guessBetterFormat(file, encoding, bestFormat);
+            }
+        }
+        return bestFormat;
+    }
 
-                while (true) {
-                    String betterFormat = null;
-
-                    List<FormatGuesser> guessers = ImportingManager.formatToGuessers.get(bestFormat);
-                    if (guessers != null) {
-                        for (FormatGuesser guesser : guessers) {
-                            betterFormat = guesser.guess(file, encoding, bestFormat);
-                            if (betterFormat != null) {
-                                break;
-                            }
+    static String guessBetterFormat(File file, String fileEncoding, String bestFormat) {
+        if (bestFormat != null && file != null) {
+            while (true) {
+                String betterFormat = null;
+
+                List<FormatGuesser> guessers = ImportingManager.formatToGuessers.get(bestFormat);
+                if (guessers != null) {
+                    for (FormatGuesser guesser : guessers) {
+                        betterFormat = guesser.guess(file, fileEncoding, bestFormat);
+                        if (betterFormat != null) {
+                            break;
                         }
                     }
+                }
 
-                    if (betterFormat != null && !betterFormat.equals(bestFormat)) {
-                        bestFormat = betterFormat;
-                    } else {
-                        break;
-                    }
+                if (betterFormat != null && !betterFormat.equals(bestFormat)) {
+                    bestFormat = betterFormat;
+                } else {
+                    break;
                 }
             }
+
         }
         return bestFormat;
     }


Test patch:
diff --git a/main/tests/server/src/com/google/refine/importing/ImportingUtilitiesTests.java b/main/tests/server/src/com/google/refine/importing/ImportingUtilitiesTests.java
index 1d1a65f37..aa75b61c8 100644
--- a/main/tests/server/src/com/google/refine/importing/ImportingUtilitiesTests.java
+++ b/main/tests/server/src/com/google/refine/importing/ImportingUtilitiesTests.java
@@ -41,7 +41,6 @@ import java.io.IOException;
 import java.io.InputStream;
 import java.io.InputStreamReader;
 import java.net.URISyntaxException;
-import java.net.URL;
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.nio.file.Paths;
@@ -69,6 +68,7 @@ import org.apache.commons.fileupload.FileUploadException;
 import org.apache.commons.io.FileSystem;
 import org.apache.commons.io.FileUtils;
 import org.apache.hc.client5.http.ClientProtocolException;
+import org.apache.hc.client5.http.entity.mime.FileBody;
 import org.apache.hc.client5.http.entity.mime.MultipartEntityBuilder;
 import org.apache.hc.client5.http.entity.mime.StringBody;
 import org.apache.hc.core5.http.ContentType;
@@ -80,7 +80,9 @@ import org.testng.annotations.Test;
 import com.google.refine.ProjectMetadata;
 import com.google.refine.importers.ImporterTest;
 import com.google.refine.importers.ImportingParserBase;
+import com.google.refine.importers.LineBasedFormatGuesser;
 import com.google.refine.importers.SeparatorBasedImporter;
+import com.google.refine.importers.TextFormatGuesser;
 import com.google.refine.importing.ImportingUtilities.Progress;
 import com.google.refine.util.JSONUtilities;
 import com.google.refine.util.ParsingUtilities;
@@ -92,6 +94,8 @@ public class ImportingUtilitiesTests extends ImporterTest {
     @BeforeMethod
     public void setUp() {
         super.setUp();
+        ImportingManager.registerFormatGuesser("text", new TextFormatGuesser());
+        ImportingManager.registerFormatGuesser("text/line-based", new LineBasedFormatGuesser());
     }
 
     @Test
@@ -576,4 +580,90 @@ public class ImportingUtilitiesTests extends ImporterTest {
 
         assertEquals(fileName, ImportingUtilities.getFileName(fileRecord));
     }
+
+    @Test
+    public void testFormatForMultipleCSVFiles() throws IOException, FileUploadException {
+        testMultipleFiles("birds", "food.small", ".csv", ContentType.create("text/csv"), "text/line-based/*sv");
+    }
+
+    @Test
+    public void testFormatForMultipleTSVFiles() throws IOException, FileUploadException {
+        testMultipleFiles("movies", "presidents", ".tsv", ContentType.create("text/tsv"), "text/line-based/*sv");
+    }
+
+    @Test
+    public void testFormatForMultipleExcelFiles() throws IOException, FileUploadException {
+        testMultipleFiles("dates", "excel95", ".xls",
+                ContentType.create("application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"), "binary");
+    }
+
+    @Test
+    public void testFormatForMultipleJSONFiles() throws IOException, FileUploadException {
+        testMultipleFiles("grid_small", "grid_small", ".json", ContentType.create("text/json"), "text/json");
+    }
+
+    private void testMultipleFiles(String file1, String file2, String fileSuffix, ContentType contentType, String expectedFormat)
+            throws IOException, FileUploadException {
+
+        String filepath1 = ClassLoader.getSystemResource(file1 + fileSuffix).getPath();
+        String filepath2 = ClassLoader.getSystemResource(file2 + fileSuffix).getPath();
+
+        File tmp1 = File.createTempFile("openrefine-test-" + file1, fileSuffix, job.getRawDataDir());
+        tmp1.deleteOnExit();
+
+        FileUtils.copyFile(new File(filepath1), tmp1);
+
+        File tmp2 = File.createTempFile("openrefine-test-" + file2, fileSuffix, job.getRawDataDir());
+        tmp2.deleteOnExit();
+
+        FileUtils.copyFile(new File(filepath2), tmp2);
+
+        MultipartEntityBuilder builder = MultipartEntityBuilder.create();
+        FileBody fileBody1 = new FileBody(tmp1, contentType);
+        FileBody fileBody2 = new FileBody(tmp2, contentType);
+        builder = builder.addPart("upload", fileBody1);
+        builder = builder.addPart("upload", fileBody2);
+
+        HttpEntity entity = builder.build();
+
+        ByteArrayOutputStream os = new ByteArrayOutputStream();
+        entity.writeTo(os);
+        ByteArrayInputStream is = new ByteArrayInputStream(os.toByteArray());
+
+        HttpServletRequest req = mock(HttpServletRequest.class);
+        when(req.getContentType()).thenReturn(entity.getContentType());
+        when(req.getMethod()).thenReturn("POST");
+        when(req.getContentLength()).thenReturn((int) entity.getContentLength());
+        when(req.getInputStream()).thenReturn(new MockServletInputStream(is));
+
+        ImportingJob job = ImportingManager.createJob();
+        Map<String, String> parameters = ParsingUtilities.parseParameters(req);
+        ObjectNode config = ParsingUtilities.mapper.createObjectNode();
+        ObjectNode retrievalRecord = ParsingUtilities.mapper.createObjectNode();
+        JSONUtilities.safePut(config, "retrievalRecord", retrievalRecord);
+        JSONUtilities.safePut(config, "state", "loading-raw-data");
+
+        final ObjectNode progress = ParsingUtilities.mapper.createObjectNode();
+        JSONUtilities.safePut(config, "progress", progress);
+
+        ImportingUtilities.retrieveContentFromPostRequest(req, parameters, job.getRawDataDir(), retrievalRecord,
+                new ImportingUtilities.Progress() {
+
+                    @Override
+                    public void setProgress(String message, int percent) {
+                        if (message != null) {
+                            JSONUtilities.safePut(progress, "message", message);
+                        }
+                        JSONUtilities.safePut(progress, "percent", percent);
+                    }
+
+                    @Override
+                    public boolean isCanceled() {
+                        return job.canceled;
+                    }
+                });
+
+        assertEquals(expectedFormat, JSONUtilities.getArray(retrievalRecord, "files").get(0).get("format").asText());
+        assertEquals(expectedFormat, JSONUtilities.getArray(retrievalRecord, "files").get(1).get("format").asText());
+    }
 }