Repository: TobikoData/sqlmesh
Pull request #3090 (2024-09-05 00:05:01)

Chore: Add retries on engine adapter integration tests
It's been observed that the engine adapter integration tests can be a bit flaky. Often this is caused by transient network issues.

Rather than trying to catch specific errors and determine which ones may or may not be retryable (and add retry logic within each engine adapter), a quick win could be to use `pytest-retry` to try the selected tests again if they fail.

Note that this was already enabled for the Trino tests, this PR just extends it to the other adapters


Patch:
diff --git a/Makefile b/Makefile
index 8ef4d722..ce6a1df8 100644
--- a/Makefile
+++ b/Makefile
@@ -167,45 +167,45 @@ engine-%-down:
 ##################
 
 clickhouse-test: engine-clickhouse-up
-	pytest -n auto -x -m "clickhouse" --junitxml=test-results/junit-clickhouse.xml
+	pytest -n auto -x -m "clickhouse" --retries 3 --junitxml=test-results/junit-clickhouse.xml
 
 clickhouse-cluster-test: engine-clickhouse-up
-	pytest -n auto -x -m "clickhouse_cluster" --junitxml=test-results/junit-clickhouse-cluster.xml
+	pytest -n auto -x -m "clickhouse_cluster" --retries 3 --junitxml=test-results/junit-clickhouse-cluster.xml
 
 duckdb-test: engine-duckdb-install
-	pytest -n auto -x -m "duckdb" --junitxml=test-results/junit-duckdb.xml
+	pytest -n auto -x -m "duckdb" --retries 3 --junitxml=test-results/junit-duckdb.xml
 
 mssql-test: engine-mssql-up
-	pytest -n auto -x -m "mssql" --junitxml=test-results/junit-mssql.xml
+	pytest -n auto -x -m "mssql" --retries 3 --junitxml=test-results/junit-mssql.xml
 
 mysql-test: engine-mysql-up
-	pytest -n auto -x -m "mysql" --junitxml=test-results/junit-mysql.xml
+	pytest -n auto -x -m "mysql" --retries 3 --junitxml=test-results/junit-mysql.xml
 
 postgres-test: engine-postgres-up
-	pytest -n auto -x -m "postgres" --junitxml=test-results/junit-postgres.xml
+	pytest -n auto -x -m "postgres" --retries 3 --junitxml=test-results/junit-postgres.xml
 
 spark-test: engine-spark-up
-	pytest -n auto -x -m "spark or pyspark" --junitxml=test-results/junit-spark.xml
+	pytest -n auto -x -m "spark or pyspark" --retries 3 --junitxml=test-results/junit-spark.xml
 
 trino-test: engine-trino-up
-	pytest -n auto -x -m "trino or trino_iceberg or trino_delta" --retries 2 --junitxml=test-results/junit-trino.xml
+	pytest -n auto -x -m "trino or trino_iceberg or trino_delta" --retries 3 --junitxml=test-results/junit-trino.xml
 
 #################
 # Cloud Engines #
 #################
 
 snowflake-test: guard-SNOWFLAKE_ACCOUNT guard-SNOWFLAKE_WAREHOUSE guard-SNOWFLAKE_DATABASE guard-SNOWFLAKE_USER guard-SNOWFLAKE_PASSWORD engine-snowflake-install
-	pytest -n auto -x -m "snowflake" --junitxml=test-results/junit-snowflake.xml
+	pytest -n auto -x -m "snowflake" --retries 3 --junitxml=test-results/junit-snowflake.xml
 
 bigquery-test: guard-BIGQUERY_KEYFILE engine-bigquery-install
-	pytest -n auto -x -m "bigquery" --junitxml=test-results/junit-bigquery.xml
+	pytest -n auto -x -m "bigquery" --retries 3 --junitxml=test-results/junit-bigquery.xml
 
 databricks-test: guard-DATABRICKS_CATALOG guard-DATABRICKS_SERVER_HOSTNAME guard-DATABRICKS_HTTP_PATH guard-DATABRICKS_ACCESS_TOKEN guard-DATABRICKS_CONNECT_VERSION engine-databricks-install
 	pip install 'databricks-connect==${DATABRICKS_CONNECT_VERSION}'
-	pytest -n auto -x -m "databricks" --junitxml=test-results/junit-databricks.xml
+	pytest -n auto -x -m "databricks" --retries 3 --junitxml=test-results/junit-databricks.xml
 
 redshift-test: guard-REDSHIFT_HOST guard-REDSHIFT_USER guard-REDSHIFT_PASSWORD guard-REDSHIFT_DATABASE engine-redshift-install
-	pytest -n auto -x -m "redshift" --junitxml=test-results/junit-redshift.xml
+	pytest -n auto -x -m "redshift" --retries 3 --junitxml=test-results/junit-redshift.xml
 
 clickhouse-cloud-test: guard-CLICKHOUSE_CLOUD_HOST guard-CLICKHOUSE_CLOUD_USERNAME guard-CLICKHOUSE_CLOUD_PASSWORD engine-clickhouse-install
-	pytest -n auto -x -m "clickhouse_cloud" --junitxml=test-results/junit-clickhouse-cloud.xml
\ No newline at end of file
+	pytest -n auto -x -m "clickhouse_cloud" --retries 3 --junitxml=test-results/junit-clickhouse-cloud.xml
\ No newline at end of file
diff --git a/docs/reference/cli.md b/docs/reference/cli.md
index e1957f0e..d3d26139 100644
--- a/docs/reference/cli.md
+++ b/docs/reference/cli.md
@@ -287,7 +287,8 @@ Options:
   --no-gaps                       Ensure that new snapshots have no data gaps
                                   when comparing to existing snapshots for
                                   matching models in the target environment.
-  --skip-backfill                 Skip the backfill step.
+  --skip-backfill, --dry-run      Skip the backfill step and only create a
+                                  virtual update for the plan.
   --forward-only                  Create a plan for forward-only changes.
   --allow-destructive-model TEXT  Allow destructive forward-only changes to
                                   models whose names match the expression.
diff --git a/docs/reference/notebook.md b/docs/reference/notebook.md
index 5b34da79..02be8a84 100644
--- a/docs/reference/notebook.md
+++ b/docs/reference/notebook.md
@@ -95,7 +95,7 @@ options:
 %plan [--start START] [--end END] [--execution-time EXECUTION_TIME]
             [--create-from CREATE_FROM] [--skip-tests]
             [--restate-model [RESTATE_MODEL ...]] [--no-gaps]
-            [--skip-backfill] [--forward-only]
+            [--skip-backfill, --dry-run] [--forward-only]
             [--effective-from EFFECTIVE_FROM] [--no-prompts] [--auto-apply]
             [--no-auto-categorization] [--include-unmodified]
             [--select-model [SELECT_MODEL ...]]
@@ -127,7 +127,8 @@ options:
   --no-gaps, -g         Ensure that new snapshots have no data gaps when
                         comparing to existing snapshots for matching models in
                         the target environment.
-  --skip-backfill       Skip the backfill step.
+  --skip-backfill, --dry-run
+                        Skip the backfill step and only create a virtual update for the plan.
   --forward-only        Create a plan for forward-only changes.
   --effective-from EFFECTIVE_FROM
                         The effective date from which to apply forward-only
diff --git a/sqlmesh/cli/main.py b/sqlmesh/cli/main.py
index 411e2a9e..dff6f194 100644
--- a/sqlmesh/cli/main.py
+++ b/sqlmesh/cli/main.py
@@ -303,8 +303,9 @@ def diff(ctx: click.Context, environment: t.Optional[str] = None) -> None:
 )
 @click.option(
     "--skip-backfill",
+    "--dry-run",
     is_flag=True,
-    help="Skip the backfill step.",
+    help="Skip the backfill step and only create a virtual update for the plan.",
 )
 @click.option(
     "--forward-only",
diff --git a/sqlmesh/magics.py b/sqlmesh/magics.py
index 51652fc0..c2fe264f 100644
--- a/sqlmesh/magics.py
+++ b/sqlmesh/magics.py
@@ -325,8 +325,9 @@ class SQLMeshMagics(Magics):
     )
     @argument(
         "--skip-backfill",
+        "--dry-run",
         action="store_true",
-        help="Skip the backfill step.",
+        help="Skip the backfill step and only create a virtual update for the plan.",
     )
     @argument(
         "--forward-only",


Test patch:
diff --git a/tests/cli/test_cli.py b/tests/cli/test_cli.py
index 309727c0..823b557c 100644
--- a/tests/cli/test_cli.py
+++ b/tests/cli/test_cli.py
@@ -214,13 +214,12 @@ def test_plan_restate_model(runner, tmp_path):
     assert_backfill_success(result)
 
 
-def test_plan_skip_backfill(runner, tmp_path):
+@pytest.mark.parametrize("flag", ["--skip-backfill", "--dry-run"])
+def test_plan_skip_backfill(runner, tmp_path, flag):
     create_example_project(tmp_path)
 
     # plan for `prod` errors if `--skip-backfill` is passed without --no-gaps
-    result = runner.invoke(
-        cli, ["--log-file-dir", tmp_path, "--paths", tmp_path, "plan", "--skip-backfill"]
-    )
+    result = runner.invoke(cli, ["--log-file-dir", tmp_path, "--paths", tmp_path, "plan", flag])
     assert result.exit_code == 1
     assert (
         "Error: When targeting the production environment either the backfill should not be skipped or the lack of data gaps should be enforced (--no-gaps flag)."
@@ -231,7 +230,7 @@ def test_plan_skip_backfill(runner, tmp_path):
     # Input: `y` to perform virtual update
     result = runner.invoke(
         cli,
-        ["--log-file-dir", tmp_path, "--paths", tmp_path, "plan", "--skip-backfill", "--no-gaps"],
+        ["--log-file-dir", tmp_path, "--paths", tmp_path, "plan", flag, "--no-gaps"],
         input="y\n",
     )
     assert result.exit_code == 0
@@ -562,15 +561,16 @@ def test_run_no_prod(runner, tmp_path):
     assert "Error: Environment 'prod' was not found." in result.output
 
 
+@pytest.mark.parametrize("flag", ["--skip-backfill", "--dry-run"])
 @freeze_time(FREEZE_TIME)
-def test_run_dev(runner, tmp_path):
+def test_run_dev(runner, tmp_path, flag):
     create_example_project(tmp_path)
 
     # Create dev environment but DO NOT backfill
     # Input: `y` for virtual update
     runner.invoke(
         cli,
-        ["--log-file-dir", tmp_path, "--paths", tmp_path, "plan", "dev", "--skip-backfill"],
+        ["--log-file-dir", tmp_path, "--paths", tmp_path, "plan", "dev", flag],
         input="y\n",
     )