Repository: metabase/metabase
Pull request #57392: Quarantine cypress flakes in download format test + small correctness fix
Author: login: noahmoss
Labels: name: no-backport
description: Do not backport this PR to any branch
name: .Team/DashViz
description: DEPRECATED  Please use .Team/UXWest instead
Created: 2025-04-29T15:18:08Z | Merged: 2025-04-29T21:38:16Z

Attempted to fix [VIZ-919](https://linear.app/metabase/issue/VIZ-919/[flaky-test]-[e2e-tests]-scenarios-question-download-scenarios) — flaky test introduced by https://github.com/metabase/metabase/pull/57155

There's some sort of race condition between loading the user's preferred download format after refresh and Cypress opening the download modal. I'm still not sure not totally sure why there's a race here because I've intercepted the GET request and I'm calling `cy.wait("@fetchFormat")` before we do anything else. Adding an extra 500ms wait here seems to resolve the flakes locally, and I've validated that this fixes the flakes with an e2e stress test: https://github.com/metabase/metabase/actions/runs/14738953531/job/41371940477

But arbitrarily pausing the tests isn't great so we've decided to quarantine them as flaky instead for the time being. This isn't a critical thing to test e2e, just a small QoL improvement, so it should be OK to make it non-blocking for CI


Diff:
diff --git a/e2e/test/scenarios/sharing/downloads/downloads.cy.spec.js b/e2e/test/scenarios/sharing/downloads/downloads.cy.spec.js
index 35c0b9bb9ee8..c1ceb23c1396 100644
--- a/e2e/test/scenarios/sharing/downloads/downloads.cy.spec.js
+++ b/e2e/test/scenarios/sharing/downloads/downloads.cy.spec.js
@@ -172,125 +172,100 @@ describe("scenarios > question > download", () => {
     );
   });
 
-  it("should remember the selected format across page reloads", () => {
-    H.createQuestion(
-      {
-        name: "Format Preference Test",
-        query: {
-          "source-table": ORDERS_ID,
-          limit: 5,
-        },
-        display: "table",
-      },
-      { visitQuestion: true },
-    );
-
-    cy.intercept(
-      "GET",
-      "/api/user-key-value/namespace/last_download_format/key/download_format_preference",
-    ).as("fetchFormat");
-
-    cy.intercept(
-      "PUT",
-      "/api/user-key-value/namespace/last_download_format/key/download_format_preference",
-    ).as("saveFormat");
-
-    cy.findByRole("button", { name: "Download results" }).click();
-    H.popover().within(() => {
-      cy.findByText(".xlsx").click();
-    });
-
-    cy.wait("@saveFormat");
-
-    cy.reload();
+  describe("download format preference", { tags: "@flaky" }, () => {
+    it("should remember the selected format across page reloads", () => {
+      cy.intercept(
+        "PUT",
+        "/api/user-key-value/namespace/last_download_format/key/download_format_preference",
+      ).as("saveFormat");
 
-    cy.wait("@fetchFormat");
-    cy.findByRole("button", { name: "Download results" }).click();
-    H.popover().within(() => {
-      cy.get("[data-checked='true']").should("contain", ".xlsx");
-    });
-
-    H.popover().within(() => {
-      cy.findByText(".json").click();
-    });
+      H.createQuestion(
+        {
+          name: "Format Preference Test",
+          query: {
+            "source-table": ORDERS_ID,
+            limit: 5,
+          },
+          display: "table",
+        },
+        { visitQuestion: true },
+      );
 
-    cy.wait("@saveFormat");
+      cy.findByRole("button", { name: "Download results" }).click();
+      H.popover().within(() => {
+        cy.findByText(".xlsx")
+          .should("be.visible")
+          .click()
+          .then(() => {
+            cy.wait("@saveFormat");
+          });
+      });
 
-    cy.reload();
+      cy.intercept(
+        "GET",
+        "/api/user-key-value/namespace/last_download_format/key/download_format_preference",
+      ).as("fetchFormat");
 
-    cy.wait("@fetchFormat");
-    cy.findByRole("button", { name: "Download results" }).click();
-    H.popover().within(() => {
-      cy.get("[data-checked='true']").should("contain", ".json");
+      cy.reload();
+      cy.wait("@fetchFormat");
+      cy.findByRole("button", { name: "Download results" }).click();
+      H.popover().within(() => {
+        cy.get("[data-checked='true']").should("contain", ".xlsx");
+      });
     });
-  });
-
-  it("should remember the download format on dashboards", () => {
-    H.createQuestion({
-      name: "Dashboard Format Test",
-      query: {
-        "source-table": ORDERS_ID,
-        limit: 5,
-      },
-      display: "table",
-    }).then(({ body: { id: questionId } }) => {
-      H.createDashboard().then(({ body: { id: dashboardId } }) => {
-        H.addOrUpdateDashboardCard({
-          card_id: questionId,
-          dashboard_id: dashboardId,
-        });
-
-        H.visitDashboard(dashboardId);
-
-        cy.intercept(
-          "GET",
-          "/api/user-key-value/namespace/last_download_format/key/download_format_preference",
-        ).as("fetchFormat");
-
-        cy.intercept(
-          "PUT",
-          "/api/user-key-value/namespace/last_download_format/key/download_format_preference",
-        ).as("saveFormat");
 
-        H.getDashboardCard(0).realHover();
-        H.getDashboardCardMenu(0).click();
-        H.popover().within(() => {
-          cy.findByText("Download results").click();
-        });
-        H.popover().within(() => {
-          cy.findByText(".xlsx").click();
-        });
+    it("should remember the download format on dashboards", () => {
+      cy.intercept(
+        "PUT",
+        "/api/user-key-value/namespace/last_download_format/key/download_format_preference",
+      ).as("saveFormat");
 
-        cy.wait("@saveFormat");
-
-        cy.reload();
+      H.createQuestion({
+        name: "Dashboard Format Test",
+        query: {
+          "source-table": ORDERS_ID,
+          limit: 5,
+        },
+        display: "table",
+      }).then(({ body: { id: questionId } }) => {
+        H.createDashboard().then(({ body: { id: dashboardId } }) => {
+          H.addOrUpdateDashboardCard({
+            card_id: questionId,
+            dashboard_id: dashboardId,
+          });
 
-        cy.wait("@fetchFormat");
-        H.getDashboardCard(0).realHover();
-        H.getDashboardCardMenu(0).click();
-        H.popover().within(() => {
-          cy.findByText("Download results").click();
-        });
-        H.popover().within(() => {
-          cy.get("[data-checked='true']").should("contain", ".xlsx");
-        });
+          H.visitDashboard(dashboardId);
 
-        H.popover().within(() => {
-          cy.findByText(".csv").click();
-        });
+          H.getDashboardCard(0).realHover();
+          H.getDashboardCardMenu(0).click();
+          H.popover().within(() => {
+            cy.findByText("Download results").click();
+          });
+          H.popover().within(() => {
+            cy.findByText(".xlsx")
+              .should("be.visible")
+              .click()
+              .then(() => {
+                cy.wait("@saveFormat");
+              });
+          });
 
-        cy.wait("@saveFormat");
+          cy.intercept(
+            "GET",
+            "/api/user-key-value/namespace/last_download_format/key/download_format_preference",
+          ).as("fetchFormat");
 
-        cy.reload();
+          cy.reload();
+          cy.wait("@fetchFormat");
 
-        cy.wait("@fetchFormat");
-        H.getDashboardCard(0).realHover();
-        H.getDashboardCardMenu(0).click();
-        H.popover().within(() => {
-          cy.findByText("Download results").click();
-        });
-        H.popover().within(() => {
-          cy.get("[data-checked='true']").should("contain", ".csv");
+          H.getDashboardCard(0).realHover();
+          H.getDashboardCardMenu(0).click();
+          H.popover().within(() => {
+            cy.findByText("Download results").click();
+          });
+          H.popover().within(() => {
+            cy.get("[data-checked='true']").should("contain", ".xlsx");
+          });
         });
       });
     });
diff --git a/frontend/src/metabase/query_builder/components/QuestionDownloadWidget/QuestionDownloadWidget.tsx b/frontend/src/metabase/query_builder/components/QuestionDownloadWidget/QuestionDownloadWidget.tsx
index 8e91de9e3c1d..51371e680154 100644
--- a/frontend/src/metabase/query_builder/components/QuestionDownloadWidget/QuestionDownloadWidget.tsx
+++ b/frontend/src/metabase/query_builder/components/QuestionDownloadWidget/QuestionDownloadWidget.tsx
@@ -97,7 +97,8 @@ export const QuestionDownloadWidget = ({
         last_table_download_format:
           newFormat !== "png"
             ? newFormat
-            : (formatPreference.last_table_download_format as TableExportFormat),
+            : (formatPreference?.last_table_download_format as TableExportFormat) ||
+              "csv",
       };
       setFormatPreference(newPreference);
     }


Reviews:
user: login: noahmoss
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-04-29T16:27:38Z
user: login: iethree
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-04-29T19:33:12Z
user: login: alxnddr
state: APPROVED
author_association: MEMBER
submitted_at: 2025-04-29T19:59:40Z

Review comments:
diff_hunk: @@ -97,7 +97,8 @@ export const QuestionDownloadWidget = ({
         last_table_download_format:
           newFormat !== "png"
             ? newFormat
-            : (formatPreference.last_table_download_format as TableExportFormat),
+            : (formatPreference?.last_table_download_format as TableExportFormat) ||
+              "csv",
path: frontend/src/metabase/query_builder/components/QuestionDownloadWidget/QuestionDownloadWidget.tsx
user: login: noahmoss
body: small correctness fix — ensure we fall back to CSV here if `last_table_download_format` was not already set
created_at: 2025-04-29T16:27:38Z
author_association: MEMBER
diff_hunk: @@ -185,47 +190,37 @@ describe("scenarios > question > download", () => {
       { visitQuestion: true },
     );
 
+    cy.findByRole("button", { name: "Download results" }).click();
+    H.popover().within(() => {
+      cy.findByText(".xlsx")
+        .should("be.visible")
+        .click()
+        .then(() => {
+          cy.wait("@saveFormat");
+        });
+    });
+
     cy.intercept(
       "GET",
       "/api/user-key-value/namespace/last_download_format/key/download_format_preference",
     ).as("fetchFormat");
 
+    cy.reload().then(() => {
+      cy.wait("@fetchFormat").then(() => {
+        cy.findByRole("button", { name: "Download results" }).click();
+        H.popover().within(() => {
+          cy.get("[data-checked='true']").should("contain", ".xlsx");
+        });
+      });
+    });
+  });
path: e2e/test/scenarios/sharing/downloads/downloads.cy.spec.js
user: login: iethree
body: I don't think these nested `.then()`s should change anything about the behavior. By default cypress does all of these things synchronously
created_at: 2025-04-29T19:33:12Z
author_association: CONTRIBUTOR

PR comments:
user: login: noahmoss
created_at: 2025-04-29T15:18:33Z
body: * **#57392** <a href="https://app.graphite.dev/github/pr/metabase/metabase/57392?utm_source=stack-comment-icon" target="_blank"><img src="https://static.graphite.dev/graphite-32x32-black.png" alt="Graphite" width="10px" height="10px"/></a> 👈 <a href="https://app.graphite.dev/github/pr/metabase/metabase/57392?utm_source=stack-comment-view-in-graphite" target="_blank">(View in Graphite)</a>
* `master`


<h2></h2>

This stack of pull requests is managed by <a href="https://graphite.dev?utm-source=stack-comment"><b>Graphite</b></a>. Learn more about <a href="https://stacking.dev/?utm_source=stack-comment">stacking</a>.
<!-- Current dependencies on/for this PR: -->
author_association: MEMBER
user: login: trunk-io[bot]
created_at: 2025-04-29T16:36:15Z
body: <!-- Trunk Test Analytics -->


[![Static Badge](https://raster.shields.io/badge/14394_tests-f56bfedf-purple)](https://app.trunk.io/metabase/flaky-tests/pr/57392?repo=metabase/metabase&commitHash=f56bfedfbe5254faf40700c2480707d5663e14eb&utm_source=gh-comment) &ensp; [![Static Badge](https://raster.shields.io/badge/0-failed-crimson)](https://app.trunk.io/metabase/flaky-tests/pr/57392?repo=metabase/metabase&commitHash=f56bfedfbe5254faf40700c2480707d5663e14eb&utm_source=gh-comment&runConclusion=FAILURE) &ensp; [![Static Badge](https://raster.shields.io/badge/1-flaky-red)](https://app.trunk.io/metabase/flaky-tests/pr/57392?repo=metabase/metabase&commitHash=f56bfedfbe5254faf40700c2480707d5663e14eb&utm_source=gh-comment&runConclusion=FAILURE) &ensp; [![Static Badge](https://raster.shields.io/badge/0-quarantined-yellow)](https://app.trunk.io/metabase/flaky-tests/pr/57392?repo=metabase/metabase&commitHash=f56bfedfbe5254faf40700c2480707d5663e14eb&utm_source=gh-comment&runConclusion=QUARANTINED)


| Flaky Test | Failure Summary | Logs |
| --- | --- | --- |
| <code>converting question to a native query (metabase\#15946, metabase\#32121, metabase\#38181, metabase\#40557) should work for both simple and nested q...</code> | The expected text was not found in the expected time, containing 'Small Marble Shoes' was not found in the received text. | [Logs ↗︎](https://app.trunk.io/metabase/flaky-tests/pr/57392?repo=metabase/metabase&commitHash=f56bfedfbe5254faf40700c2480707d5663e14eb&utm_source=gh-comment&runConclusion=FAILURE&runId=cb837c0b-5760-56bc-9a9d-5f375ce0b84d) |
<sub>

[View Full Report ↗︎](https://app.trunk.io/metabase/flaky-tests/pr/57392?repo=metabase/metabase&commitHash=f56bfedfbe5254faf40700c2480707d5663e14eb&utm_source=gh-comment) ⋅ [Docs](https://docs.trunk.io/flaky-tests?utm_source=gh-comment)

</sub>
author_association: NONE
user: login: github-actions[bot]
created_at: 2025-04-29T21:05:42Z
body: 
  ## e2e tests failed on `f56bfedfbe5254faf40700c2480707d5663e14eb-1`

  [e2e test run](https://github.com/metabase/metabase/actions/runs/14741112639?pr=57392)

  File | Test Name
  ---- | ---------
  `notebook-native-preview-sidebar.cy.spec.ts` | converting question to a native query (metabase#15946, metabase#32121, metabase#38181, metabase#40557) > should work for both simple and nested questions based on previously converted GUI query

author_association: CONTRIBUTOR