Repository: OpenRefine/OpenRefine (java)

Issue:
Move Cypress Cloud secret to Github Secrets
We are currently publishing the secret for Cypress cloud in our workflow file, so anyone who clones the repo can push to our dashboard, making for confusing debugging and wasted time.  We should invalidate that secret, mint a new one, move it to Github secrets, and adjust the workflow to reference it from the secrets store

https://github.com/OpenRefine/OpenRefine/issues/6942#issuecomment-2442785321_


Pull request:
ci: Move Cypress record key to a GH Actions secret
Closes #6959.

I haven't invalidated the old token yet, it should be done after merging this PR. I believe existing PRs will then start using the new workflow, if not they can be rebased.

Patch:
diff --git a/.github/workflows/snapshot_release.yml b/.github/workflows/snapshot_release.yml
index 97fdad8e8..46b8b0fd9 100644
--- a/.github/workflows/snapshot_release.yml
+++ b/.github/workflows/snapshot_release.yml
@@ -42,11 +42,6 @@ jobs:
           ref: ${{ github.event.pull_request.head.ref }}
           repository: ${{ github.event.pull_request.head.repo.full_name }}
 
-      - name: Set up secrets
-        run: |
-          echo "CYPRESS_RECORD_KEY=$(echo YzE3ZDU4OGItZTBkOC00ZjJmLTg4NjYtNzJmNmFmYmRhNGQxCg== | base64 -d)" >> $GITHUB_ENV
-          echo "CYPRESS_PROJECT_ID=s5du3k" >> $GITHUB_ENV
-
       - name: Set up Java 11
         uses: actions/setup-java@v4
         with:
@@ -87,8 +82,8 @@ jobs:
           ./refine e2e_tests
         env:
           CYPRESS_BROWSER: ${{ matrix.browser }}
-          CYPRESS_RECORD_KEY: ${{ env.CYPRESS_RECORD_KEY }}
-          CYPRESS_PROJECT_ID: ${{ env.CYPRESS_PROJECT_ID }}
+          CYPRESS_RECORD_KEY: ${{ secrets.CYPRESS_RECORD_KEY }}
+          CYPRESS_PROJECT_ID: ${{ vars.CYPRESS_PROJECT_ID }}
           CYPRESS_CI_BUILD_ID: '${{ github.run_id }}'
           CYPRESS_SPECS: ${{ matrix.specs.paths }}
           CYPRESS_GROUP: ${{ matrix.specs.group }}
diff --git a/benchmark/pom.xml b/benchmark/pom.xml
index 720be7997..c3c649056 100644
--- a/benchmark/pom.xml
+++ b/benchmark/pom.xml
@@ -141,7 +141,7 @@
         </plugin>
         <plugin>
           <artifactId>maven-site-plugin</artifactId>
-          <version>3.20.0</version>
+          <version>3.21.0</version>
         </plugin>
         <plugin>
           <artifactId>maven-source-plugin</artifactId>
diff --git a/extensions/database/pom.xml b/extensions/database/pom.xml
index 223271ac5..55b2d2507 100644
--- a/extensions/database/pom.xml
+++ b/extensions/database/pom.xml
@@ -109,12 +109,12 @@
     <dependency>
       <groupId>org.mariadb.jdbc</groupId>
       <artifactId>mariadb-java-client</artifactId>
-      <version>3.4.1</version>
+      <version>3.5.0</version>
     </dependency>
     <dependency>
       <groupId>org.xerial</groupId>
       <artifactId>sqlite-jdbc</artifactId>
-      <version>3.46.1.3</version>
+      <version>3.47.0.0</version>
     </dependency>
     <dependency>
       <groupId>com.fasterxml.jackson.core</groupId>
diff --git a/modules/core/src/main/java/com/google/refine/importing/ImportingUtilities.java b/modules/core/src/main/java/com/google/refine/importing/ImportingUtilities.java
index 0e4898964..7942f67e3 100644
--- a/modules/core/src/main/java/com/google/refine/importing/ImportingUtilities.java
+++ b/modules/core/src/main/java/com/google/refine/importing/ImportingUtilities.java
@@ -301,7 +301,6 @@ public class ImportingUtilities {
                     JSONUtilities.safePut(fileRecord, "origin", "clipboard");
                     JSONUtilities.safePut(fileRecord, "declaredEncoding", encoding);
                     JSONUtilities.safePut(fileRecord, "declaredMimeType", (String) null);
-                    JSONUtilities.safePut(fileRecord, "format", "text");
                     JSONUtilities.safePut(fileRecord, "fileName", "(clipboard)");
                     JSONUtilities.safePut(fileRecord, "location", getRelativePath(file, rawDataDir));
 
@@ -309,6 +308,7 @@ public class ImportingUtilities {
                             calculateProgressPercent(update.totalExpectedSize, update.totalRetrievedSize));
 
                     JSONUtilities.safePut(fileRecord, "size", saveStreamToFile(stream, file, null));
+                    JSONUtilities.safePut(fileRecord, "format", guessBetterFormat(file, fileRecord));
                     JSONUtilities.append(fileRecords, fileRecord);
 
                     clipboardCount++;
@@ -439,7 +439,6 @@ public class ImportingUtilities {
                             calculateProgressPercent(update.totalExpectedSize, update.totalRetrievedSize));
 
                     JSONUtilities.safePut(fileRecord, "size", saveStreamToFile(stream, file, null));
-                    JSONUtilities.safePut(fileRecord, "format", guessBetterFormat(file, request.getCharacterEncoding(), "text"));
                     // TODO: This needs to be refactored to be able to test import from archives
                     if (postProcessRetrievedFile(rawDataDir, file, fileRecord, fileRecords, progress)) {
                         archiveCount++;
@@ -700,10 +699,7 @@ public class ImportingUtilities {
 
     static public void postProcessSingleRetrievedFile(File file, ObjectNode fileRecord) {
         if (!fileRecord.has("format")) {
-            JSONUtilities.safePut(fileRecord, "format",
-                    ImportingManager.getFormat(
-                            file.getName(),
-                            JSONUtilities.getString(fileRecord, "declaredMimeType", null)));
+            JSONUtilities.safePut(fileRecord, "format", guessBetterFormat(file, fileRecord));
         }
     }
 
@@ -1048,6 +1044,13 @@ public class ImportingUtilities {
         return bestFormat;
     }
 
+    static String guessBetterFormat(File file, ObjectNode fileRecord) {
+        String encoding = JSONUtilities.getString(fileRecord, "declaredEncoding", null);
+        String bestFormat = ImportingManager.getFormat(file.getName(), JSONUtilities.getString(fileRecord, "declaredMimeType", null));
+        bestFormat = bestFormat == null ? "text" : bestFormat;
+        return guessBetterFormat(file, encoding, bestFormat);
+    }
+
     static void rankFormats(ImportingJob job, final String bestFormat, ArrayNode rankedFormats) {
         final Map<String, String[]> formatToSegments = new HashMap<String, String[]>();
 
diff --git a/pom.xml b/pom.xml
index 53724fe57..8c4f6e554 100644
--- a/pom.xml
+++ b/pom.xml
@@ -101,8 +101,8 @@
     <odfdom-java.version>0.12.0</odfdom-java.version> <!-- do not update to 0.10.0, see issue #4397 -->
     <vicino.version>1.2.1</vicino.version>
     <clojure.version>1.12.0</clojure.version>
-    <httpclient5.version>5.4</httpclient5.version>
-    <httpcore5.version>5.3</httpcore5.version>
+    <httpclient5.version>5.4.1</httpclient5.version>
+    <httpcore5.version>5.3.1</httpcore5.version>
     <univocity-csv.version>2.9.1</univocity-csv.version>
     <swc-parser-lazy.version>3.1.9</swc-parser-lazy.version>
     <commons-io.version>2.17.0</commons-io.version>
@@ -119,8 +119,8 @@
     <build-helper-maven-plugin.version>3.6.0</build-helper-maven-plugin.version>
     <maven-compiler-plugin.version>3.13.0</maven-compiler-plugin.version>
     <maven-resources-plugin.version>3.3.1</maven-resources-plugin.version>
-    <exec-maven-plugin.version>3.4.1</exec-maven-plugin.version>
-    <maven-dependency-plugin.version>3.8.0</maven-dependency-plugin.version>
+    <exec-maven-plugin.version>3.5.0</exec-maven-plugin.version>
+    <maven-dependency-plugin.version>3.8.1</maven-dependency-plugin.version>
     <jacoco-maven-plugin.version>0.8.12</jacoco-maven-plugin.version>
     <surefire.version>3.5.1</surefire.version>
     <maven-jar-plugin.version>3.4.2</maven-jar-plugin.version>


Test patch:
diff --git a/.github/workflows/pull_request_e2e.yml b/.github/workflows/pull_request_e2e.yml
index fb09561a5..0b0810892 100644
--- a/.github/workflows/pull_request_e2e.yml
+++ b/.github/workflows/pull_request_e2e.yml
@@ -40,11 +40,6 @@ jobs:
     steps:
       - uses: actions/checkout@v4
 
-      - name: Set up secrets
-        run: |
-          echo "CYPRESS_RECORD_KEY=$(echo YzE3ZDU4OGItZTBkOC00ZjJmLTg4NjYtNzJmNmFmYmRhNGQxCg== | base64 -d)" >> $GITHUB_ENV
-          echo "CYPRESS_PROJECT_ID=s5du3k" >> $GITHUB_ENV
-
       - name: Set up Java 17
         uses: actions/setup-java@v4
         with:
@@ -85,8 +80,8 @@ jobs:
           ./refine e2e_tests
         env:
           CYPRESS_BROWSER: ${{ matrix.browser }}
-          CYPRESS_RECORD_KEY: ${{ env.CYPRESS_RECORD_KEY }}
-          CYPRESS_PROJECT_ID: ${{ env.CYPRESS_PROJECT_ID }}
+          CYPRESS_RECORD_KEY: ${{ secrets.CYPRESS_RECORD_KEY }}
+          CYPRESS_PROJECT_ID: ${{ vars.CYPRESS_PROJECT_ID }}
           CYPRESS_CI_BUILD_ID: '${{ github.run_id }}'
           CYPRESS_SPECS: ${{ matrix.specs.paths }}
           CYPRESS_GROUP: ${{ matrix.specs.group }}
diff --git a/main/tests/cypress/cypress/e2e/project/grid/column/facet/facets.cy.js b/main/tests/cypress/cypress/e2e/project/grid/column/facet/facets.cy.js
index 8d94a1985..3b191f161 100644
--- a/main/tests/cypress/cypress/e2e/project/grid/column/facet/facets.cy.js
+++ b/main/tests/cypress/cypress/e2e/project/grid/column/facet/facets.cy.js
@@ -4,25 +4,23 @@
  */
 
 /*
-   IMPORTANT: Any test reliant on Facet Actions (Edit,Include/Exclude) should wait 100 msec to ensure the event
+   IMPORTANT: Any test reliant on Facet Actions (Edit,Include/Exclude) should wait >100 msec to ensure the event
    handlers are setup, otherwise clicking on the action will do nothing.
    Ref:  window.setTimeout(wireEvents, 100); in list-facet.js
  */
-const clickFacetAction = (facetType, text, action, setVisibility = false) => {
+const clickFacetAction = (facetType, text, action) => {
   cy.getFacetContainer(facetType)
-      .contains(text)
-      .parent()
-      .within(() => {
-        const elem = cy.contains(action);
-
-        if (setVisibility) {
-          elem.invoke('css', 'visibility', 'visible')
-              .should('have.css', 'visibility', 'visible')
-              .click();
-        } else {
-          elem.click();
-        }
-      });
+    .contains(text)
+    .parent()
+    .invoke('trigger', 'mouseenter'); // include/exclude not visible until we mouse over
+  // start a new chain after the trigger
+  cy.getFacetContainer(facetType)
+    .contains(text)
+    .siblings()
+    .get(".facet-choice-toggle")
+    .contains(action)
+    .should('have.css', 'visibility', 'visible') // partially occluded, so "be.visible" won't work
+    .click();
 };
 
 describe(__filename, function () {
@@ -226,28 +224,28 @@ describe(__filename, function () {
 
     // Wait to ensure action click handler is setup
     // Ref:  window.setTimeout(wireEvents, 100); in list-facet.js
-    cy.wait(100);
-    clickFacetAction('Shrt_Desc','ALLSPICE,GROUND', 'include', true);
+    cy.wait(150);
+    clickFacetAction('Shrt_Desc','ALLSPICE,GROUND', 'include');
     cy.getCell(0, 'Shrt_Desc').should('contain', 'ALLSPICE,GROUND');
     cy.get('#tool-panel').contains('1 matching rows');
 
     // Wait to ensure action click handler is setup
     // Ref:  window.setTimeout(wireEvents, 100); in list-facet.js
-    cy.wait(100);
-    clickFacetAction('Shrt_Desc','ANISE SEED', 'include', true);
+    cy.wait(150);
+    clickFacetAction('Shrt_Desc','ANISE SEED', 'include');
     cy.getCell(1, 'Shrt_Desc').should('contain', 'ANISE SEED');
     cy.get('#tool-panel').contains('2 matching rows');
 
     // Wait to ensure action click handler is setup
     // Ref:  window.setTimeout(wireEvents, 100); in list-facet.js
-    cy.wait(100);
-    clickFacetAction('Shrt_Desc','BUTTER OIL,ANHYDROUS', 'include', true);
+    cy.wait(150);
+    clickFacetAction('Shrt_Desc','BUTTER OIL,ANHYDROUS', 'include');
     cy.getCell(0, 'Shrt_Desc').should('contain', 'BUTTER OIL,ANHYDROUS');
     cy.get('#tool-panel').contains('3 matching rows');
 
     // Wait to ensure action click handler is setup
     // Ref:  window.setTimeout(wireEvents, 100); in list-facet.js
-    cy.wait(100);
+    cy.wait(150);
     clickFacetAction('Shrt_Desc','ALLSPICE,GROUND', 'exclude');
     cy.get('#tool-panel').contains('2 matching rows');
   });
@@ -258,9 +256,9 @@ describe(__filename, function () {
 
     // Wait to ensure action click handler is setup
     // Ref:  window.setTimeout(wireEvents, 100); in list-facet.js
-    cy.wait(100);
+    cy.wait(150);
     // do a basic facetting, expect 1 row
-    clickFacetAction('Shrt_Desc','ALLSPICE,GROUND','include',true);
+    clickFacetAction('Shrt_Desc','ALLSPICE,GROUND','include');
     cy.getCell(0, 'Shrt_Desc').should('to.contain', 'ALLSPICE,GROUND');
     cy.get('#tool-panel').contains('1 matching rows');
 
@@ -285,7 +283,7 @@ describe(__filename, function () {
 
     // Wait to ensure action click handler is setup
     // Ref:  window.setTimeout(wireEvents, 100); in list-facet.js
-    cy.wait(100);
+    cy.wait(150);
     // do a basic facetting, expect 1 row
     clickFacetAction('Shrt_Desc','ALLSPICE,GROUND','include',true);
     cy.get('#tool-panel').contains('1 matching rows');
@@ -342,7 +340,7 @@ describe(__filename, function () {
 
     // Wait to ensure action click handler is setup
     // Ref:  window.setTimeout(wireEvents, 100); in list-facet.js
-    cy.wait(100);
+    cy.wait(150);
 
     cy.get('div.facet-body-inner > div:nth-child(8)')
         .contains('15.87')
@@ -380,27 +378,15 @@ describe(__filename, function () {
     );
   });
 
-  // // This test is unstable, mouseover behavior is unpredictable
-  // // This might be because the element is detached from the DOM in the middle
-  // // it('Test include/exlude toggle', function () {
-  // // 	cy.loadAndVisitProject('food.small');
-  // // 	cy.columnActionClick('NDB_No', ['Facet', 'Text facet']);
-  // // 	cy.getFacetContainer('NDB_No').find('.facet-choice[choiceindex="0"]').trigger('mouseover');
-  // // 	cy.getFacetContainer('NDB_No').find('.facet-choice[choiceindex="0"] a.facet-choice-toggle').contains('include').should('be.visible');
-  // // 	cy.getFacetContainer('NDB_No').find('.facet-choice[choiceindex="0"] a.facet-choice-toggle').click();
-  // // 	cy.getFacetContainer('NDB_No').find('.facet-choice[choiceindex="0"] a.facet-choice-toggle').contains('exclude');
-  // // 	cy.getFacetContainer('NDB_No').find('.facet-choice[choiceindex="0"] a.facet-choice-toggle').click();
-  // // 	cy.getFacetContainer('NDB_No').find('.facet-choice[choiceindex="0"] a.facet-choice-toggle').contains('include');
-  // // });
-
-  // it('Test collapsing facet panels', function () {
-  // The following test does not work
-  // Because the facet panels uses soem weird CSS with overflow:hidden, Cypress can not detect it
-  // //// # cy.loadAndVisitProject('food.small');
-  // //// # cy.columnActionClick('NDB_No', ['Facet', 'Text facet']);
-  // //// # ensure facet inner panel is visible
-  // //// # cy.get('#refine-tabs-facets .facets-container li:nth-child(1) .facet-body-inner').should('be.visible');
-  // //// #collapse the panel
-  // //// # cy.get('#refine-tabs-facets .facets-container li:nth-child(1) a[bind="minimizeButton"]').click();
-  // ////
+  it('Test collapsing facet panels', function () {
+    cy.loadAndVisitProject('food.small');
+    cy.columnActionClick('NDB_No', ['Facet', 'Text facet']);
+    // ensure facet inner panel is visible
+    cy.get('#refine-tabs-facets .facets-container li:nth-child(1) .facet-body-inner').should('be.visible');
+    // collapse the panel
+    cy.get('#refine-tabs-facets .facets-container li:nth-child(1) a[bind="minimizeButton"]').click();
+    // Make sure the body is hidden
+    cy.get('#refine-tabs-facets .facets-container li:nth-child(1) .facet-body').should('not.be.visible');
+  });
+
 });
diff --git a/main/tests/server/src/com/google/refine/importing/ImportingUtilitiesTests.java b/main/tests/server/src/com/google/refine/importing/ImportingUtilitiesTests.java
index aa75b61c8..a44686a12 100644
--- a/main/tests/server/src/com/google/refine/importing/ImportingUtilitiesTests.java
+++ b/main/tests/server/src/com/google/refine/importing/ImportingUtilitiesTests.java
@@ -94,8 +94,7 @@ public class ImportingUtilitiesTests extends ImporterTest {
     @BeforeMethod
     public void setUp() {
         super.setUp();
-        ImportingManager.registerFormatGuesser("text", new TextFormatGuesser());
-        ImportingManager.registerFormatGuesser("text/line-based", new LineBasedFormatGuesser());
+        importFlowSettings();
     }
 
     @Test
@@ -594,7 +593,7 @@ public class ImportingUtilitiesTests extends ImporterTest {
     @Test
     public void testFormatForMultipleExcelFiles() throws IOException, FileUploadException {
         testMultipleFiles("dates", "excel95", ".xls",
-                ContentType.create("application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"), "binary");
+                ContentType.create("application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"), "binary/text/xml/xls/xlsx");
     }
 
     @Test
@@ -602,6 +601,11 @@ public class ImportingUtilitiesTests extends ImporterTest {
         testMultipleFiles("grid_small", "grid_small", ".json", ContentType.create("text/json"), "text/json");
     }
 
+    @Test
+    public void testFormatForMultipleODSFiles() throws IOException, FileUploadException {
+        testMultipleFiles("films", "films", ".ods", ContentType.create("application/vnd.oasis.opendocument.spreadsheet"), "text/xml/ods");
+    }
+
     private void testMultipleFiles(String file1, String file2, String fileSuffix, ContentType contentType, String expectedFormat)
             throws IOException, FileUploadException {
 
@@ -666,4 +670,65 @@ public class ImportingUtilitiesTests extends ImporterTest {
         assertEquals(expectedFormat, JSONUtilities.getArray(retrievalRecord, "files").get(0).get("format").asText());
         assertEquals(expectedFormat, JSONUtilities.getArray(retrievalRecord, "files").get(1).get("format").asText());
     }
+
+    private void importFlowSettings() {
+        // Register Format guessers
+        ImportingManager.registerFormatGuesser("text", new TextFormatGuesser());
+        ImportingManager.registerFormatGuesser("text/line-based", new LineBasedFormatGuesser());
+
+        // Extension to format mappings
+        ImportingManager.registerExtension(".txt", "text");
+        ImportingManager.registerExtension(".csv", "text/line-based/*sv");
+        ImportingManager.registerExtension(".tsv", "text/line-based/*sv");
+        ImportingManager.registerExtension(".xml", "text/xml");
+        ImportingManager.registerExtension(".atom", "text/xml");
+        ImportingManager.registerExtension(".json", "text/json");
+        ImportingManager.registerExtension(".js", "text/json");
+        ImportingManager.registerExtension(".xls", "binary/text/xml/xls/xlsx");
+        ImportingManager.registerExtension(".xlsx", "binary/text/xml/xls/xlsx");
+        ImportingManager.registerExtension(".ods", "text/xml/ods");
+        ImportingManager.registerExtension(".nt", "text/rdf/nt");
+        ImportingManager.registerExtension(".ntriples", "text/rdf/nt");
+        ImportingManager.registerExtension(".n3", "text/rdf/n3");
+        ImportingManager.registerExtension(".ttl", "text/rdf/ttl");
+        ImportingManager.registerExtension(".jsonld", "text/rdf/ld+json");
+        ImportingManager.registerExtension(".rdf", "text/rdf/xml");
+        ImportingManager.registerExtension(".marc", "text/marc");
+        ImportingManager.registerExtension(".mrc", "text/marc");
+        ImportingManager.registerExtension(".wiki", "text/wiki");
+
+        // Mime type to format mappings
+        ImportingManager.registerMimeType("text/plain", "text");
+        ImportingManager.registerMimeType("text/csv", "text/line-based/*sv");
+        ImportingManager.registerMimeType("text/x-csv", "text/line-based/*sv");
+        ImportingManager.registerMimeType("text/tab-separated-value", "text/line-based/*sv");
+        ImportingManager.registerMimeType("text/tab-separated-values", "text/line-based/*sv");
+        ImportingManager.registerMimeType("text/fixed-width", "text/line-based/fixed-width");
+        ImportingManager.registerMimeType("application/n-triples", "text/rdf/nt");
+        ImportingManager.registerMimeType("text/n3", "text/rdf/n3");
+        ImportingManager.registerMimeType("text/rdf+n3", "text/rdf/n3");
+        ImportingManager.registerMimeType("text/turtle", "text/rdf/ttl");
+        ImportingManager.registerMimeType("application/xml", "text/xml");
+        ImportingManager.registerMimeType("text/xml", "text/xml");
+        ImportingManager.registerMimeType("+xml", "text/xml"); // suffix will be tried only as fallback
+        ImportingManager.registerMimeType("application/rdf+xml", "text/rdf/xml");
+        ImportingManager.registerMimeType("application/ld+json", "text/rdf/ld+json");
+        ImportingManager.registerMimeType("application/atom+xml", "text/xml");
+        ImportingManager.registerMimeType("application/msexcel", "binary/text/xml/xls/xlsx");
+        ImportingManager.registerMimeType("application/x-msexcel", "binary/text/xml/xls/xlsx");
+        ImportingManager.registerMimeType("application/x-ms-excel", "binary/text/xml/xls/xlsx");
+        ImportingManager.registerMimeType("application/vnd.ms-excel", "binary/text/xml/xls/xlsx");
+        ImportingManager.registerMimeType("application/x-excel", "binary/text/xml/xls/xlsx");
+        ImportingManager.registerMimeType("application/xls", "binary/text/xml/xls/xlsx");
+        ImportingManager.registerMimeType("application/x-xls", "binary/text/xml/xls/xlsx");
+        ImportingManager.registerMimeType("application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", "binary/text/xml/xls/xlsx");
+        ImportingManager.registerMimeType("application/vnd.openxmlformats-officedocument.spreadsheetml.template",
+                "binary/text/xml/xls/xlsx");
+        ImportingManager.registerMimeType("application/vnd.oasis.opendocument.spreadsheet", "text/xml/ods");
+        ImportingManager.registerMimeType("application/json", "text/json");
+        ImportingManager.registerMimeType("application/javascript", "text/json");
+        ImportingManager.registerMimeType("text/json", "text/json");
+        ImportingManager.registerMimeType("+json", "text/json"); // suffix will be tried only as fallback
+        ImportingManager.registerMimeType("application/marc", "text/marc");
+    }
 }