Repository: trunk-io/analytics-cli
Pull request #546: Allow defining fallback exit code
Author: login: gnalh
Created: 2025-04-15T03:33:01Z | Merged: 2025-04-17T17:48:52Z

This mimics the behavior of `test` in `upload` by allowing users to set the exit code of the previous result.


Diff:
diff --git a/cli-tests/src/command_builder.rs b/cli-tests/src/command_builder.rs
index 9c2ce278..e2ee920e 100644
--- a/cli-tests/src/command_builder.rs
+++ b/cli-tests/src/command_builder.rs
@@ -22,6 +22,7 @@ pub struct UploadArgs {
     disable_quarantining: Option<bool>,
     allow_empty_test_results: Option<bool>,
     variant: Option<String>,
+    test_process_exit_code: Option<i32>,
 }
 
 impl UploadArgs {
@@ -41,6 +42,7 @@ impl UploadArgs {
             disable_quarantining: None,
             allow_empty_test_results: None,
             variant: None,
+            test_process_exit_code: None,
         }
     }
 
@@ -162,6 +164,16 @@ impl UploadArgs {
                 .into_iter()
                 .flat_map(|variant: String| vec![String::from("--variant"), variant]),
         )
+        .chain(
+            self.test_process_exit_code
+                .into_iter()
+                .flat_map(|test_process_exit_code: i32| {
+                    vec![
+                        String::from("--test-process-exit-code"),
+                        test_process_exit_code.to_string(),
+                    ]
+                }),
+        )
         .collect()
     }
 }
@@ -415,6 +427,22 @@ impl<'b> CommandBuilder<'b> {
         self
     }
 
+    pub fn test_process_exit_code(&mut self, new_value: i32) -> &mut Self {
+        match &mut self.command_type {
+            CommandType::Upload { upload_args, .. } => {
+                upload_args.test_process_exit_code = Some(new_value);
+            }
+            CommandType::Quarantine { upload_args, .. } => {
+                upload_args.test_process_exit_code = Some(new_value);
+            }
+            CommandType::Test { upload_args, .. } => {
+                upload_args.test_process_exit_code = Some(new_value);
+            }
+            CommandType::Validate { .. } => {}
+        }
+        self
+    }
+
     pub fn command(&self) -> Command {
         let mut command = Command::new(CARGO_RUN.path());
         let args = self.build_args();
diff --git a/cli-tests/src/upload.rs b/cli-tests/src/upload.rs
index 96a99355..d4674baa 100644
--- a/cli-tests/src/upload.rs
+++ b/cli-tests/src/upload.rs
@@ -276,6 +276,50 @@ async fn upload_bundle_success_status_code() {
     println!("{assert}");
 }
 
+// same test as upload_bundle_success_status_code but with a previous exit code set
+#[tokio::test(flavor = "multi_thread")]
+async fn upload_bundle_success_preceding_failure() {
+    let temp_dir = tempdir().unwrap();
+    generate_mock_git_repo(&temp_dir);
+    let test_bep_path = get_test_file_path("test_fixtures/bep_retries");
+    // The test cases need not match up or have timestamps, so long as there is a testSummary
+    // That indicates a flake or pass
+    let uri_fail = format!(
+        "file://{}",
+        get_test_file_path("../cli/test_fixtures/junit1_fail.xml")
+    );
+    let uri_pass = format!(
+        "file://{}",
+        get_test_file_path("../cli/test_fixtures/junit0_pass.xml")
+    );
+
+    let bep_content = fs::read_to_string(&test_bep_path)
+        .unwrap()
+        .replace("${URI_FAIL}", &uri_fail)
+        .replace("${URI_PASS}", &uri_pass);
+    let bep_path = temp_dir.path().join("bep.json");
+    fs::write(&bep_path, bep_content).unwrap();
+
+    let state = MockServerBuilder::new().spawn_mock_server().await;
+    let test_process_exit_code = 127;
+
+    // Even though the junits contain failures, they contain retries that succeeded,
+    // so the upload command should have a successful exit code
+    let assert = CommandBuilder::upload(temp_dir.path(), state.host.clone())
+        .test_process_exit_code(test_process_exit_code)
+        .command()
+        .assert()
+        .code(test_process_exit_code)
+        .failure();
+
+    // No quarantine request
+    let requests = state.requests.lock().unwrap().clone();
+    assert_eq!(requests.len(), 3);
+
+    // HINT: View CLI output with `cargo test -- --nocapture`
+    println!("{assert}");
+}
+
 #[tokio::test(flavor = "multi_thread")]
 async fn upload_bundle_success_timestamp_status_code() {
     let temp_dir = tempdir().unwrap();
diff --git a/cli/src/context.rs b/cli/src/context.rs
index bb78136e..5447083f 100644
--- a/cli/src/context.rs
+++ b/cli/src/context.rs
@@ -315,7 +315,7 @@ pub async fn gather_exit_code_and_quarantined_tests_context(
     disable_quarantining: bool,
     api_client: &ApiClient,
     file_set_builder: &FileSetBuilder,
-    test_run_result: &Option<TestRunResult>,
+    default_exit_code: Option<i32>,
 ) -> i32 {
     // Run the quarantine step and update the exit code.
     let failed_tests_extractor = FailedTestsExtractor::new(
@@ -332,9 +332,9 @@ pub async fn gather_exit_code_and_quarantined_tests_context(
             },
     } = if disable_quarantining {
         // use the exit code of the test run result if exists
-        if let Some(test_run_result) = test_run_result {
+        if let Some(exit_code) = default_exit_code {
             QuarantineContext {
-                exit_code: test_run_result.exit_code,
+                exit_code,
                 quarantine_status: QuarantineBulkTestStatus {
                     quarantine_results: failed_tests_extractor
                         .failed_tests()
@@ -366,7 +366,7 @@ pub async fn gather_exit_code_and_quarantined_tests_context(
             },
             file_set_builder,
             Some(failed_tests_extractor),
-            test_run_result.as_ref().map(|t| t.exit_code),
+            default_exit_code,
         )
         .await
     };
diff --git a/cli/src/upload_command.rs b/cli/src/upload_command.rs
index ad9b764d..2e86421b 100644
--- a/cli/src/upload_command.rs
+++ b/cli/src/upload_command.rs
@@ -130,6 +130,14 @@ pub struct UploadArgs {
         hide = true
     )]
     pub variant: Option<String>,
+    #[arg(
+        long,
+        help = "The exit code to use when not all tests are quarantined.",
+        required = false,
+        num_args = 1,
+        hide = true
+    )]
+    pub test_process_exit_code: Option<i32>,
 }
 
 impl UploadArgs {
@@ -172,6 +180,7 @@ pub async fn run_upload(
     } else {
         chrono::Utc::now().into()
     };
+
     let api_client = ApiClient::new(&upload_args.token, &upload_args.org_url_slug)?;
 
     let PreTestContext {
@@ -218,13 +227,17 @@ pub async fn run_upload(
             }
         }
     }
-
+    let default_exit_code = if let Some(exit_code) = upload_args.test_process_exit_code {
+        Some(exit_code)
+    } else {
+        test_run_result.as_ref().map(|r| r.exit_code)
+    };
     let exit_code = gather_exit_code_and_quarantined_tests_context(
         &mut meta,
         upload_args.disable_quarantining || !upload_args.use_quarantining,
         &api_client,
         &file_set_builder,
-        &test_run_result,
+        default_exit_code,
     )
     .await;
 


Reviews:
user: login: gnalh
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-04-16T18:02:23Z
user: login: cmillar-trunk
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-04-16T18:26:51Z
user: login: gnalh
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-04-16T18:30:24Z
user: login: gnalh
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-04-16T18:32:23Z
user: login: cmillar-trunk
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-04-16T18:58:41Z
user: login: cmillar-trunk
state: APPROVED
author_association: CONTRIBUTOR
submitted_at: 2025-04-17T17:13:11Z

Review comments:
diff_hunk: @@ -130,6 +130,14 @@ pub struct UploadArgs {
         hide = true
     )]
     pub variant: Option<String>,
+    #[arg(
+        long,
+        help = "The exit code to use when not all tests are quarantined.",
+        required = false,
+        num_args = 1,
+        hide = true
+    )]
+    pub previous_exit_code: Option<i32>,
path: cli/src/upload_command.rs
user: login: gnalh
body: Open to a better name here. I had `fallback_exit_code` and `default_exit_code`, but didn't like either option. It's currently hidden while we experiment. 
created_at: 2025-04-16T18:02:08Z
author_association: CONTRIBUTOR
diff_hunk: @@ -130,6 +130,14 @@ pub struct UploadArgs {
         hide = true
     )]
     pub variant: Option<String>,
+    #[arg(
+        long,
+        help = "The exit code to use when not all tests are quarantined.",
+        required = false,
+        num_args = 1,
+        hide = true
+    )]
+    pub previous_exit_code: Option<i32>,
path: cli/src/upload_command.rs
user: login: cmillar-trunk
body: Perhaps `override_exit_code_with`, since it's more explicit about what you should expect to have happen when you set it? (If not, I have to admit I prefer either option in your comment over `previous`, just because previous is describing it in the context of a workflow run, and not describing what it's for).
created_at: 2025-04-16T18:24:16Z
author_association: CONTRIBUTOR
diff_hunk: @@ -377,7 +377,7 @@ pub async fn gather_exit_code_and_quarantined_tests_context(
             },
             file_set_builder,
             Some(failed_tests_extractor),
-            test_run_result.as_ref().map(|t| t.exit_code),
+            default_exit_code,
path: cli/src/context.rs
user: login: cmillar-trunk
body: Should this `orElse` to the `test_run_result.exit_code`, so that we still use that if the default_exit_code isn't set?
created_at: 2025-04-16T18:26:04Z
author_association: CONTRIBUTOR
diff_hunk: @@ -377,7 +377,7 @@ pub async fn gather_exit_code_and_quarantined_tests_context(
             },
             file_set_builder,
             Some(failed_tests_extractor),
-            test_run_result.as_ref().map(|t| t.exit_code),
+            default_exit_code,
path: cli/src/context.rs
user: login: gnalh
body: No, this doesn't need the `test_run_result` and never really did. That is and should be handled higher up in the call stack.
created_at: 2025-04-16T18:30:24Z
author_association: CONTRIBUTOR
diff_hunk: @@ -130,6 +130,14 @@ pub struct UploadArgs {
         hide = true
     )]
     pub variant: Option<String>,
+    #[arg(
+        long,
+        help = "The exit code to use when not all tests are quarantined.",
+        required = false,
+        num_args = 1,
+        hide = true
+    )]
+    pub previous_exit_code: Option<i32>,
path: cli/src/upload_command.rs
user: login: gnalh
body: >just because previous is describing it in the context of a workflow run

But isn't that exactly what this is for? We want to allow users to propagate the exit code or result of a preceding job from their test run when they are not wrapping the underlying command.
created_at: 2025-04-16T18:32:23Z
author_association: CONTRIBUTOR
diff_hunk: @@ -130,6 +130,14 @@ pub struct UploadArgs {
         hide = true
     )]
     pub variant: Option<String>,
+    #[arg(
+        long,
+        help = "The exit code to use when not all tests are quarantined.",
+        required = false,
+        num_args = 1,
+        hide = true
+    )]
+    pub previous_exit_code: Option<i32>,
path: cli/src/upload_command.rs
user: login: cmillar-trunk
body: That's one thing it can be used for, and our intended use for it, but saying 'this is the previous code' doesn't tell us about how it's used - if someone is looking at arguments to solve a problem where they want to modify the exit code, they need to trawl through help because `previous_exit_code` does not tell you the behaviour of this flag. They need to infer that `previous` -> 'overrides a standard exit code' , or else trawl through the help for all arguments. It's harder to discover, and trawling through the whole help text is not a thing people will do, as thousands of stackoverflow questions about using bash commands demonstrate.
created_at: 2025-04-16T18:58:41Z
author_association: CONTRIBUTOR

PR comments:
user: login: trunk-io[bot]
created_at: 2025-04-15T03:33:05Z
body: 😎 Merged successfully - [details](https://app.trunk.io/gh/trunk-io/analytics-cli/merge-queue/main/546).
author_association: NONE
user: login: trunk-staging-io[bot]
created_at: 2025-04-15T04:07:05Z
body: <!-- Trunk Test Analytics -->


[![Static Badge](https://raster.shields.io/badge/785_tests-19a0bef0-purple)](https://app.trunk-staging.io/trunk-staging-org/flaky-tests/pr/546?repo=trunk-io/analytics-cli&commitHash=19a0bef01c85e37f8eea09e2447520d2264e71fc&utm_source=gh-comment) &ensp; [![Static Badge](https://raster.shields.io/badge/0-failed-crimson)](https://app.trunk-staging.io/trunk-staging-org/flaky-tests/pr/546?repo=trunk-io/analytics-cli&commitHash=19a0bef01c85e37f8eea09e2447520d2264e71fc&utm_source=gh-comment&runConclusion=FAILURE) &ensp; [![Static Badge](https://raster.shields.io/badge/0-flaky-red)](https://app.trunk-staging.io/trunk-staging-org/flaky-tests/pr/546?repo=trunk-io/analytics-cli&commitHash=19a0bef01c85e37f8eea09e2447520d2264e71fc&utm_source=gh-comment&runConclusion=FAILURE) &ensp; [![Static Badge](https://raster.shields.io/badge/0-quarantined-yellow)](https://app.trunk-staging.io/trunk-staging-org/flaky-tests/pr/546?repo=trunk-io/analytics-cli&commitHash=19a0bef01c85e37f8eea09e2447520d2264e71fc&utm_source=gh-comment&runConclusion=QUARANTINED)


<sub>

[View Full Report ↗︎](https://app.trunk-staging.io/trunk-staging-org/flaky-tests/pr/546?repo=trunk-io/analytics-cli&commitHash=19a0bef01c85e37f8eea09e2447520d2264e71fc&utm_source=gh-comment) ⋅ [Docs](https://docs.trunk.io/flaky-tests?utm_source=gh-comment)

</sub>
author_association: NONE
user: login: trunk-staging-io[bot]
created_at: 2025-04-15T04:07:05Z
body: <!-- Trunk Test Analytics -->


[![Static Badge](https://raster.shields.io/badge/781_tests-be16be73-purple)](https://app.trunk-staging.io/trunk-staging-org/flaky-tests/pr/546?repo=trunk-io/analytics-cli&commitHash=be16be73f86e9d65e8ccc809a526d3eb2daea393&utm_source=gh-comment) &ensp; [![Static Badge](https://raster.shields.io/badge/0-failed-crimson)](https://app.trunk-staging.io/trunk-staging-org/flaky-tests/pr/546?repo=trunk-io/analytics-cli&commitHash=be16be73f86e9d65e8ccc809a526d3eb2daea393&utm_source=gh-comment&runConclusion=FAILURE) &ensp; [![Static Badge](https://raster.shields.io/badge/0-flaky-red)](https://app.trunk-staging.io/trunk-staging-org/flaky-tests/pr/546?repo=trunk-io/analytics-cli&commitHash=be16be73f86e9d65e8ccc809a526d3eb2daea393&utm_source=gh-comment&runConclusion=FAILURE) &ensp; [![Static Badge](https://raster.shields.io/badge/0-quarantined-yellow)](https://app.trunk-staging.io/trunk-staging-org/flaky-tests/pr/546?repo=trunk-io/analytics-cli&commitHash=be16be73f86e9d65e8ccc809a526d3eb2daea393&utm_source=gh-comment&runConclusion=QUARANTINED)


<sub>

[View Full Report ↗︎](https://app.trunk-staging.io/trunk-staging-org/flaky-tests/pr/546?repo=trunk-io/analytics-cli&commitHash=be16be73f86e9d65e8ccc809a526d3eb2daea393&utm_source=gh-comment) ⋅ [Docs](https://docs.trunk.io/flaky-tests?utm_source=gh-comment)

</sub>
author_association: NONE
user: login: trunk-staging-io[bot]
created_at: 2025-04-15T04:07:05Z
body: <!-- Trunk Test Analytics -->


[![Static Badge](https://raster.shields.io/badge/781_tests-be16be73-purple)](https://app.trunk-staging.io/trunk-staging-org/flaky-tests/pr/546?repo=trunk-io/analytics-cli&commitHash=be16be73f86e9d65e8ccc809a526d3eb2daea393&utm_source=gh-comment) &ensp; [![Static Badge](https://raster.shields.io/badge/0-failed-crimson)](https://app.trunk-staging.io/trunk-staging-org/flaky-tests/pr/546?repo=trunk-io/analytics-cli&commitHash=be16be73f86e9d65e8ccc809a526d3eb2daea393&utm_source=gh-comment&runConclusion=FAILURE) &ensp; [![Static Badge](https://raster.shields.io/badge/0-flaky-red)](https://app.trunk-staging.io/trunk-staging-org/flaky-tests/pr/546?repo=trunk-io/analytics-cli&commitHash=be16be73f86e9d65e8ccc809a526d3eb2daea393&utm_source=gh-comment&runConclusion=FAILURE) &ensp; [![Static Badge](https://raster.shields.io/badge/0-quarantined-yellow)](https://app.trunk-staging.io/trunk-staging-org/flaky-tests/pr/546?repo=trunk-io/analytics-cli&commitHash=be16be73f86e9d65e8ccc809a526d3eb2daea393&utm_source=gh-comment&runConclusion=QUARANTINED)


<sub>

[View Full Report ↗︎](https://app.trunk-staging.io/trunk-staging-org/flaky-tests/pr/546?repo=trunk-io/analytics-cli&commitHash=be16be73f86e9d65e8ccc809a526d3eb2daea393&utm_source=gh-comment) ⋅ [Docs](https://docs.trunk.io/flaky-tests?utm_source=gh-comment)

</sub>
author_association: NONE
user: login: trunk-io[bot]
created_at: 2025-04-15T04:15:17Z
body: <!-- Trunk Test Analytics -->


[![Static Badge](https://raster.shields.io/badge/772_tests-19a0bef0-purple)](https://app.trunk.io/trunk/flaky-tests/pr/546?repo=trunk-io/analytics-cli&commitHash=19a0bef01c85e37f8eea09e2447520d2264e71fc&utm_source=gh-comment) &ensp; [![Static Badge](https://raster.shields.io/badge/0-failed-crimson)](https://app.trunk.io/trunk/flaky-tests/pr/546?repo=trunk-io/analytics-cli&commitHash=19a0bef01c85e37f8eea09e2447520d2264e71fc&utm_source=gh-comment&runConclusion=FAILURE) &ensp; [![Static Badge](https://raster.shields.io/badge/0-flaky-red)](https://app.trunk.io/trunk/flaky-tests/pr/546?repo=trunk-io/analytics-cli&commitHash=19a0bef01c85e37f8eea09e2447520d2264e71fc&utm_source=gh-comment&runConclusion=FAILURE) &ensp; [![Static Badge](https://raster.shields.io/badge/0-quarantined-yellow)](https://app.trunk.io/trunk/flaky-tests/pr/546?repo=trunk-io/analytics-cli&commitHash=19a0bef01c85e37f8eea09e2447520d2264e71fc&utm_source=gh-comment&runConclusion=QUARANTINED)


<sub>

[View Full Report ↗︎](https://app.trunk.io/trunk/flaky-tests/pr/546?repo=trunk-io/analytics-cli&commitHash=19a0bef01c85e37f8eea09e2447520d2264e71fc&utm_source=gh-comment) ⋅ [Docs](https://docs.trunk.io/flaky-tests?utm_source=gh-comment)

</sub>
author_association: NONE