Repository: prefix-dev/rattler-build (rust)

Issue:
turn off `set -x` for test scripts
rattler-build is still inserting `set -x` in testing scripts which is causing failures for some test scripts. This should be turned off by default in all cases.

Pull request description:
feat: improve debug mode a bit and add it to `test` subcommand
This enables `rattler-build test --package-file foobar.tar.bz2 --debug`

I want to refactor the other debug subcommand a bit as well and perhaps remove it in favor of just running `build` / `test` with the `--debug` flag.

Fixes #1639 

Also

- [x] propagate correct `--debug` flag (https://github.com/prefix-dev/rattler-build/issues/1639)
- [x] create debug environment in `output` folder instead of `/tmp/` (https://github.com/prefix-dev/rattler-build/issues/1204 & https://github.com/prefix-dev/rattler-build/issues/1436)
- [x] Make sure to create proper prefix with `conda-meta/history` file so that Conda detects it (https://github.com/prefix-dev/rattler-build/issues/1643)
- [x] move packages when test fail (https://github.com/prefix-dev/rattler-build/issues/1003)

Patch:
diff --git a/docs/reference/cli.md b/docs/reference/cli.md
index d80532a4..dbb8ecb3 100644
--- a/docs/reference/cli.md
+++ b/docs/reference/cli.md
@@ -308,6 +308,16 @@ These test files are written at "package creation time" and are part of the pack
 	The number of threads to use for compression
 
 
+- `--test-index <TEST_INDEX>`
+
+	The index of the test to run. This is used to run a specific test from the package
+
+
+- `--debug`
+
+	Build test environment and output debug information for manual debugging
+
+
 - `--experimental`
 
 	Enable experimental features
diff --git a/py-rattler-build/rattler_build/__init__.py b/py-rattler-build/rattler_build/__init__.py
index 9793a980..5dc5eb8e 100644
--- a/py-rattler-build/rattler_build/__init__.py
+++ b/py-rattler-build/rattler_build/__init__.py
@@ -55,6 +55,7 @@ def build_recipes(
     noarch_build_platform: Union[str, None] = None,
     allow_insecure_host: Union[List[str], None] = None,
     continue_on_failure: bool = False,
+    debug: bool = False,
 ) -> None:
     """
     Build packages from a list of recipes.
@@ -83,7 +84,8 @@ def build_recipes(
         skip_existing: Whether to skip packages that already exist in any channel. If set to `none`, do not skip any packages, default when not specified. If set to `local`, only skip packages that already exist locally, default when using `--skip-existing`. If set to `all`, skip packages that already exist in any channel.
         noarch_build_platform: Define a "noarch platform" for which the noarch packages will be built for. The noarch builds will be skipped on the other platforms.
         allow_insecure_host: Allow insecure hosts for the build.
-        continue_on_failure: Continue building other recipes even if one fails.
+        continue_on_failure: Continue building other recipes even if one fails. (default: False)
+        debug: Enable or disable debug mode. (default: False)
 
     Returns:
         None
@@ -114,6 +116,7 @@ def build_recipes(
         noarch_build_platform,
         allow_insecure_host,
         continue_on_failure,
+        debug,
     )
 
 
@@ -123,6 +126,9 @@ def test_package(
     compression_threads: Union[int, None] = None,
     auth_file: Union[str, Path, None] = None,
     channel_priority: Union[str, None] = None,
+    allow_insecure_host: Union[List[str], None] = None,
+    debug: bool = False,
+    test_index: Union[int, None] = None,
 ) -> None:
     """
     Run a test for a single package.
@@ -133,11 +139,16 @@ def test_package(
         compression_threads: The number of threads to use for compression.
         auth_file: The authentication file.
         channel_priority: The channel priority.
+        allow_insecure_host: Allow insecure hosts for the build.
+        debug: Enable or disable debug mode. (default: False)
+        test_index: The test to run, selected by index. (default: None - run all tests)
 
     Returns:
         None
     """
-    test_package_py(package_file, channel, compression_threads, auth_file, channel_priority)
+    test_package_py(
+        package_file, channel, compression_threads, auth_file, channel_priority, allow_insecure_host, debug, test_index
+    )
 
 
 def upload_package_to_quetz(
diff --git a/py-rattler-build/src/lib.rs b/py-rattler-build/src/lib.rs
index 0f8f9784..41a7b2e0 100644
--- a/py-rattler-build/src/lib.rs
+++ b/py-rattler-build/src/lib.rs
@@ -2,6 +2,7 @@ use std::{path::PathBuf, str::FromStr};
 
 use ::rattler_build::{
     build_recipes, get_rattler_build_version,
+    metadata::Debug,
     opt::{
         AnacondaData, ArtifactoryData, BuildData, ChannelPriorityWrapper, CommonData,
         CondaForgeData, PrefixData, QuetzData, TestData,
@@ -24,7 +25,7 @@ fn get_rattler_build_version_py() -> PyResult<String> {
 }
 
 #[pyfunction]
-#[pyo3(signature = (recipes, up_to, build_platform, target_platform, host_platform, channel, variant_config, ignore_recipe_variants, render_only, with_solve, keep_build, no_build_id, package_format, compression_threads, io_concurrency_limit, no_include_recipe, test, output_dir, auth_file, channel_priority, skip_existing, noarch_build_platform, allow_insecure_host=None, continue_on_failure=false))]
+#[pyo3(signature = (recipes, up_to, build_platform, target_platform, host_platform, channel, variant_config, ignore_recipe_variants, render_only, with_solve, keep_build, no_build_id, package_format, compression_threads, io_concurrency_limit, no_include_recipe, test, output_dir, auth_file, channel_priority, skip_existing, noarch_build_platform, allow_insecure_host=None, continue_on_failure=false, debug=false))]
 #[allow(clippy::too_many_arguments)]
 fn build_recipes_py(
     recipes: Vec<PathBuf>,
@@ -51,6 +52,7 @@ fn build_recipes_py(
     noarch_build_platform: Option<String>,
     allow_insecure_host: Option<Vec<String>>,
     continue_on_failure: bool,
+    debug: bool,
 ) -> PyResult<()> {
     let channel_priority = channel_priority
         .map(|c| ChannelPriorityWrapper::from_str(&c).map(|c| c.value))
@@ -124,7 +126,7 @@ fn build_recipes_py(
         noarch_build_platform,
         None,
         None,
-        true,
+        Debug::new(debug),
         continue_on_failure.into(),
     );
 
@@ -137,8 +139,9 @@ fn build_recipes_py(
     })
 }
 
+#[allow(clippy::too_many_arguments)]
 #[pyfunction]
-#[pyo3(signature = (package_file, channel, compression_threads, auth_file, channel_priority, allow_insecure_host=None))]
+#[pyo3(signature = (package_file, channel, compression_threads, auth_file, channel_priority, allow_insecure_host=None, debug=false, test_index=None))]
 fn test_package_py(
     package_file: PathBuf,
     channel: Option<Vec<String>>,
@@ -146,6 +149,8 @@ fn test_package_py(
     auth_file: Option<PathBuf>,
     channel_priority: Option<String>,
     allow_insecure_host: Option<Vec<String>>,
+    debug: bool,
+    test_index: Option<usize>,
 ) -> PyResult<()> {
     let channel_priority = channel_priority
         .map(|c| ChannelPriorityWrapper::from_str(&c).map(|c| c.value))
@@ -173,7 +178,14 @@ fn test_package_py(
                 .collect::<PyResult<_>>()?,
         ),
     };
-    let test_data = TestData::new(package_file, channel, compression_threads, common);
+    let test_data = TestData::new(
+        package_file,
+        channel,
+        compression_threads,
+        Debug::new(debug),
+        test_index,
+        common,
+    );
 
     let rt = tokio::runtime::Runtime::new().unwrap();
     rt.block_on(async {
diff --git a/src/build.rs b/src/build.rs
index 013f6463..82ae55bb 100644
--- a/src/build.rs
+++ b/src/build.rs
@@ -9,6 +9,7 @@ use crate::{
     metadata::{Output, build_reindexed_channels},
     recipe::parser::TestType,
     render::solver::load_repodatas,
+    script::InterpreterError,
     tool_configuration,
 };
 
@@ -130,7 +131,18 @@ pub async fn run_build(
         .await
         .into_diagnostic()?;
 
-    output.run_build_script().await.into_diagnostic()?;
+    match output.run_build_script().await {
+        Ok(_) => {}
+        Err(InterpreterError::Debug(info)) => {
+            tracing::info!("{}", info);
+            return Err(miette::miette!(
+                "Script not executed because debug mode is enabled"
+            ));
+        }
+        Err(InterpreterError::ExecutionFailed(_)) => {
+            return Err(miette::miette!("Script failed to execute"));
+        }
+    }
 
     // Package all the new files
     let (result, paths_json) = output
diff --git a/src/lib.rs b/src/lib.rs
index 31e28faa..72b27913 100644
--- a/src/lib.rs
+++ b/src/lib.rs
@@ -557,12 +557,15 @@ pub async fn run_build_from_args(
                 to_test
             };
 
-            // let testable = can_test(&test_queue, &all_output_names, &outputs_to_build);
             for (output, archive) in &to_test {
-                package_test::run_test(
+                match package_test::run_test(
                     archive,
                     &TestConfiguration {
-                        test_prefix: output.build_configuration.directories.work_dir.join("test"),
+                        test_prefix: output
+                            .build_configuration
+                            .directories
+                            .output_dir
+                            .join("test"),
                         target_platform: Some(output.build_configuration.target_platform),
                         host_platform: Some(output.build_configuration.host_platform.clone()),
                         current_platform: output.build_configuration.build_platform.clone(),
@@ -577,11 +580,34 @@ pub async fn run_build_from_args(
                         channel_priority: tool_configuration.channel_priority,
                         solve_strategy: SolveStrategy::Highest,
                         tool_configuration: tool_configuration.clone(),
+                        test_index: None,
+                        output_dir: output.build_configuration.directories.output_dir.clone(),
+                        debug: output.build_configuration.debug,
                     },
                     None,
                 )
                 .await
-                .into_diagnostic()?;
+                {
+                    Ok(_) => {}
+                    Err(e) => {
+                        // move the package file to the failed directory
+                        let failed_dir = output
+                            .build_configuration
+                            .directories
+                            .output_dir
+                            .join("broken");
+                        fs::create_dir_all(&failed_dir).into_diagnostic()?;
+                        fs::rename(archive, failed_dir.join(archive.file_name().unwrap()))
+                            .into_diagnostic()?;
+
+                        if tool_configuration.continue_on_failure == ContinueOnFailure::Yes {
+                            tracing::error!("Test failed for {}: {}", output.identifier(), e);
+                            output.record_warning(&format!("Test failed: {}", e));
+                        } else {
+                            return Err(miette::miette!("Test failed: {}", e));
+                        }
+                    }
+                }
             }
         }
     }
@@ -687,10 +713,13 @@ pub async fn run_test(
         host_platform: None,
         current_platform,
         keep_test_prefix: false,
+        test_index: test_data.test_index,
         channels,
         channel_priority: tool_config.channel_priority,
         solve_strategy: SolveStrategy::Highest,
         tool_configuration: tool_config,
+        output_dir: test_data.common.output_dir,
+        debug: test_data.debug,
     };
 
     let package_name = package_file
diff --git a/src/opt.rs b/src/opt.rs
index 3c00a132..c8510844 100644
--- a/src/opt.rs
+++ b/src/opt.rs
@@ -503,7 +503,7 @@ impl BuildData {
         noarch_build_platform: Option<Platform>,
         extra_meta: Option<Vec<(String, Value)>>,
         sandbox_configuration: Option<SandboxConfiguration>,
-        debug: bool,
+        debug: Debug,
         continue_on_failure: ContinueOnFailure,
     ) -> Self {
         Self {
@@ -537,7 +537,7 @@ impl BuildData {
             noarch_build_platform,
             extra_meta,
             sandbox_configuration,
-            debug: Debug::new(debug),
+            debug,
             continue_on_failure,
         }
     }
@@ -582,7 +582,7 @@ impl BuildData {
             opts.noarch_build_platform,
             opts.extra_meta,
             opts.sandbox_arguments.into(),
-            opts.debug,
+            Debug::new(opts.debug),
             opts.continue_on_failure.into(),
         )
     }
@@ -622,6 +622,14 @@ pub struct TestOpts {
     #[clap(long, env = "RATTLER_COMPRESSION_THREADS")]
     pub compression_threads: Option<u32>,
 
+    /// The index of the test to run. This is used to run a specific test from the package.
+    #[clap(long)]
+    pub test_index: Option<usize>,
+
+    /// Build test environment and output debug information for manual debugging.
+    #[arg(long)]
+    pub debug: bool,
+
     /// Common options.
     #[clap(flatten)]
     pub common: CommonOpts,
@@ -634,6 +642,8 @@ pub struct TestData {
     pub package_file: PathBuf,
     pub compression_threads: Option<u32>,
     pub common: CommonData,
+    pub test_index: Option<usize>,
+    pub debug: Debug,
 }
 
 impl TestData {
@@ -644,6 +654,8 @@ impl TestData {
             value.package_file,
             value.channels,
             value.compression_threads,
+            Debug::new(value.debug),
+            value.test_index,
             CommonData::from_opts_and_config(value.common, config.unwrap_or_default()),
         )
     }
@@ -653,12 +665,16 @@ impl TestData {
         package_file: PathBuf,
         channels: Option<Vec<NamedChannelOrUrl>>,
         compression_threads: Option<u32>,
+        debug: Debug,
+        test_index: Option<usize>,
         common: CommonData,
     ) -> Self {
         Self {
             package_file,
             channels,
             compression_threads,
+            test_index,
+            debug,
             common,
         }
     }
diff --git a/src/packaging.rs b/src/packaging.rs
index e6de33d9..3ee172e9 100644
--- a/src/packaging.rs
+++ b/src/packaging.rs
@@ -214,7 +214,7 @@ fn write_recipe_folder(
         .build_configuration
         .channels
         .iter()
-        .map(|url| clean_url(url))
+        .map(clean_url)
         .map(|url| ChannelUrl::from(url.parse::<url::Url>().expect("url is valid")))
         .collect();
 
diff --git a/src/script/interpreter/bash.rs b/src/script/interpreter/bash.rs
index 085e77d4..6fc52582 100644
--- a/src/script/interpreter/bash.rs
+++ b/src/script/interpreter/bash.rs
@@ -3,14 +3,34 @@ use std::path::PathBuf;
 use rattler_conda_types::Platform;
 use rattler_shell::shell;
 
-use crate::script::{ExecutionArgs, interpreter::DEBUG_HELP, run_process_with_replacements};
+use crate::script::{ExecutionArgs, run_process_with_replacements};
 
-use super::{BASH_PREAMBLE, Interpreter, find_interpreter};
+use super::{BASH_PREAMBLE, Interpreter, InterpreterError, find_interpreter};
 
 pub(crate) struct BashInterpreter;
 
+fn print_debug_info(args: &ExecutionArgs) -> String {
+    let mut output = String::new();
+    output.push_str("\nDebug mode enabled - not executing the script.\n\n");
+    output.push_str(&format!("  Work directory: {}\n", args.work_dir.display()));
+    output.push_str(&format!("  Prefix: {}\n", args.run_prefix.display()));
+
+    if let Some(build_prefix) = &args.build_prefix {
+        output.push_str(&format!("  Build prefix: {}\n", build_prefix.display()));
+    } else {
+        output.push_str("  Build prefix: None\n");
+    }
+
+    output.push_str("\nTo run the script manually, use the following command:\n\n");
+    output.push_str(&format!("  cd {:?} && ./conda_build.sh\n\n", args.work_dir));
+    output.push_str("To run commands interactively in the build environment:\n\n");
+    output.push_str(&format!("  cd {:?} && source build_env.sh", args.work_dir));
+
+    output
+}
+
 impl Interpreter for BashInterpreter {
-    async fn run(&self, args: ExecutionArgs) -> Result<(), std::io::Error> {
+    async fn run(&self, args: ExecutionArgs) -> Result<(), InterpreterError> {
         let script = self.get_script(&args, shell::Bash).unwrap();
 
         let build_env_path = args.work_dir.join("build_env.sh");
@@ -22,6 +42,14 @@ impl Interpreter for BashInterpreter {
         let script = format!("{}\n{}", preamble, args.script.script());
         tokio::fs::write(&build_script_path, script).await?;
 
+        // Mark build_env.sh and conda_build.sh as executable
+        #[cfg(unix)]
+        {
+            use std::{fs::Permissions, os::unix::fs::PermissionsExt};
+            let permissions = Permissions::from_mode(0o755);
+            tokio::fs::set_permissions(&build_script_path, permissions).await?;
+        }
+
         let build_script_path_str = build_script_path.to_string_lossy().to_string();
         let mut cmd_args = vec!["bash", "-e"];
         if args.debug.is_enabled() {
@@ -29,6 +57,10 @@ impl Interpreter for BashInterpreter {
         }
         cmd_args.push(&build_script_path_str);
 
+        if args.debug.is_enabled() {
+            return Err(InterpreterError::Debug(print_debug_info(&args)));
+        }
+
         let output = run_process_with_replacements(
             &cmd_args,
             &args.work_dir,
@@ -40,12 +72,11 @@ impl Interpreter for BashInterpreter {
         if !output.status.success() {
             let status_code = output.status.code().unwrap_or(1);
             tracing::error!("Script failed with status {}", status_code);
-            tracing::error!("Work directory: '{}'", args.work_dir.display());
-            tracing::error!("{}", DEBUG_HELP);
-            return Err(std::io::Error::new(
+            tracing::error!("{}", print_debug_info(&args));
+            return Err(InterpreterError::ExecutionFailed(std::io::Error::new(
                 std::io::ErrorKind::Other,
                 "Script failed".to_string(),
-            ));
+            )));
         }
 
         Ok(())
diff --git a/src/script/interpreter/cmd_exe.rs b/src/script/interpreter/cmd_exe.rs
index 1216efc9..bbcf7e29 100644
--- a/src/script/interpreter/cmd_exe.rs
+++ b/src/script/interpreter/cmd_exe.rs
@@ -3,14 +3,37 @@ use std::path::PathBuf;
 use rattler_conda_types::Platform;
 use rattler_shell::shell;
 
-use crate::script::{ExecutionArgs, interpreter::DEBUG_HELP, run_process_with_replacements};
+use crate::script::{ExecutionArgs, run_process_with_replacements};
 
-use super::{CMDEXE_PREAMBLE, Interpreter, find_interpreter};
+use super::{CMDEXE_PREAMBLE, Interpreter, InterpreterError, find_interpreter};
+
+fn print_debug_info(args: &ExecutionArgs) -> String {
+    let mut output = String::new();
+    output.push_str("Debug mode enabled - not executing the script. \n\nRun the script with the following arguments:\n");
+    output.push_str(&format!("  Work directory: {}\n", args.work_dir.display()));
+    output.push_str(&format!("  Prefix: {}\n", args.run_prefix.display()));
+
+    if let Some(build_prefix) = &args.build_prefix {
+        output.push_str(&format!("  Build prefix: {}\n", build_prefix.display()));
+    } else {
+        output.push_str("  Build prefix: None\n");
+    }
+
+    output.push_str("\nTo run the script manually, use the following command:\n");
+    output.push_str(&format!(
+        "  cd {:?} && ./conda_build.bat\n\n",
+        args.work_dir
+    ));
+    output.push_str("To run commands interactively in the build environment:\n");
+    output.push_str(&format!("  cd {:?} && call build_env.bat", args.work_dir));
+
+    output
+}
 
 pub(crate) struct CmdExeInterpreter;
 
 impl Interpreter for CmdExeInterpreter {
-    async fn run(&self, args: ExecutionArgs) -> Result<(), std::io::Error> {
+    async fn run(&self, args: ExecutionArgs) -> Result<(), InterpreterError> {
         let script = self.get_script(&args, shell::CmdExe).unwrap();
 
         let build_env_path = args.work_dir.join("build_env.bat");
@@ -32,6 +55,10 @@ impl Interpreter for CmdExeInterpreter {
         let build_script_path_str = build_script_path.to_string_lossy().to_string();
         let cmd_args = ["cmd.exe", "/d", "/c", &build_script_path_str];
 
+        if args.debug.is_enabled() {
+            return Err(InterpreterError::Debug(print_debug_info(&args)));
+        }
+
         let output = run_process_with_replacements(
             &cmd_args,
             &args.work_dir,
@@ -43,12 +70,11 @@ impl Interpreter for CmdExeInterpreter {
         if !output.status.success() {
             let status_code = output.status.code().unwrap_or(1);
             tracing::error!("Script failed with status {}", status_code);
-            tracing::error!("Work directory: '{}'", args.work_dir.display());
-            tracing::error!("{}", DEBUG_HELP);
-            return Err(std::io::Error::new(
+            tracing::error!("{}", print_debug_info(&args));
+            return Err(InterpreterError::ExecutionFailed(std::io::Error::new(
                 std::io::ErrorKind::Other,
                 "Script failed".to_string(),
-            ));
+            )));
         }
 
         Ok(())
diff --git a/src/script/interpreter/mod.rs b/src/script/interpreter/mod.rs
index 5bd17cb9..623ed1fa 100644
--- a/src/script/interpreter/mod.rs
+++ b/src/script/interpreter/mod.rs
@@ -25,7 +25,17 @@ use rattler_shell::{
 
 use super::ExecutionArgs;
 
-pub(crate) const DEBUG_HELP: &str = "To debug the build, run it manually in the work directory (execute the `./conda_build.sh` or `conda_build.bat` script)";
+/// The error type for the interpreter
+#[derive(Debug, thiserror::Error)]
+pub enum InterpreterError {
+    /// This error is returned when running in debug mode
+    #[error("Debugging information: {0}")]
+    Debug(String),
+
+    /// This error is returned when the script execution fails or the interpreter is not found
+    #[error("IO Error: {0}")]
+    ExecutionFailed(#[from] std::io::Error),
+}
 
 pub const BASH_PREAMBLE: &str = r#"#!/bin/bash
 ## Start of bash preamble
@@ -110,7 +120,7 @@ pub trait Interpreter {
         Ok(shell_script.contents()?)
     }
 
-    async fn run(&self, args: ExecutionArgs) -> Result<(), std::io::Error>;
+    async fn run(&self, args: ExecutionArgs) -> Result<(), InterpreterError>;
 
     #[allow(dead_code)]
     async fn find_interpreter(
diff --git a/src/script/interpreter/nushell.rs b/src/script/interpreter/nushell.rs
index 37444143..0173c539 100644
--- a/src/script/interpreter/nushell.rs
+++ b/src/script/interpreter/nushell.rs
@@ -7,9 +7,9 @@ use rattler_shell::{
     shell::{self, Shell, ShellEnum},
 };
 
-use crate::script::{ExecutionArgs, interpreter::DEBUG_HELP, run_process_with_replacements};
+use crate::script::{ExecutionArgs, run_process_with_replacements};
 
-use super::{Interpreter, find_interpreter};
+use super::{Interpreter, InterpreterError, find_interpreter};
 
 pub(crate) struct NuShellInterpreter;
 
@@ -23,7 +23,7 @@ if not ("CONDA_BUILD" in $env) {
 "#;
 
 impl Interpreter for NuShellInterpreter {
-    async fn run(&self, args: ExecutionArgs) -> Result<(), std::io::Error> {
+    async fn run(&self, args: ExecutionArgs) -> Result<(), InterpreterError> {
         let host_shell_type = ShellEnum::default();
         let nushell = ShellEnum::NuShell(Default::default());
 
@@ -104,10 +104,10 @@ impl Interpreter for NuShellInterpreter {
             match find_interpreter("nu", args.build_prefix.as_ref(), &args.execution_platform) {
                 Ok(Some(path)) => path,
                 _ => {
-                    return Err(std::io::Error::new(
+                    return Err(InterpreterError::ExecutionFailed(std::io::Error::new(
                         std::io::ErrorKind::NotFound,
                         "NuShell executable not found in PATH",
-                    ));
+                    )));
                 }
             }
             .to_string_lossy()
@@ -127,11 +127,10 @@ impl Interpreter for NuShellInterpreter {
             let status_code = output.status.code().unwrap_or(1);
             tracing::error!("Script failed with status {}", status_code);
             tracing::error!("Work directory: '{}'", args.work_dir.display());
-            tracing::error!("{}", DEBUG_HELP);
-            return Err(std::io::Error::new(
+            return Err(InterpreterError::ExecutionFailed(std::io::Error::new(
                 std::io::ErrorKind::Other,
                 "Script failed".to_string(),
-            ));
+            )));
         }
 
         Ok(())
diff --git a/src/script/interpreter/perl.rs b/src/script/interpreter/perl.rs
index 0c4113cd..6b375339 100644
--- a/src/script/interpreter/perl.rs
+++ b/src/script/interpreter/perl.rs
@@ -4,13 +4,13 @@ use rattler_conda_types::Platform;
 
 use crate::script::{ExecutionArgs, ResolvedScriptContents};
 
-use super::{BashInterpreter, CmdExeInterpreter, Interpreter, find_interpreter};
+use super::{BashInterpreter, CmdExeInterpreter, Interpreter, InterpreterError, find_interpreter};
 
 pub(crate) struct PerlInterpreter;
 
 // Perl interpreter calls either bash or cmd.exe interpreter for activation and then runs Perl script
 impl Interpreter for PerlInterpreter {
-    async fn run(&self, args: ExecutionArgs) -> Result<(), std::io::Error> {
+    async fn run(&self, args: ExecutionArgs) -> Result<(), InterpreterError> {
         let perl_script = args.work_dir.join("conda_build_script.pl");
         tokio::fs::write(&perl_script, args.script.script()).await?;
 
diff --git a/src/script/interpreter/python.rs b/src/script/interpreter/python.rs
index 19b5f340..a0898963 100644
--- a/src/script/interpreter/python.rs
+++ b/src/script/interpreter/python.rs
@@ -4,13 +4,13 @@ use rattler_conda_types::Platform;
 
 use crate::script::{ExecutionArgs, ResolvedScriptContents};
 
-use super::{BashInterpreter, CmdExeInterpreter, Interpreter, find_interpreter};
+use super::{BashInterpreter, CmdExeInterpreter, Interpreter, InterpreterError, find_interpreter};
 
 pub(crate) struct PythonInterpreter;
 
 // python interpreter calls either bash or cmd.exe interpreter for activation and then runs python script
 impl Interpreter for PythonInterpreter {
-    async fn run(&self, args: ExecutionArgs) -> Result<(), std::io::Error> {
+    async fn run(&self, args: ExecutionArgs) -> Result<(), InterpreterError> {
         let py_script = args.work_dir.join("conda_build_script.py");
         tokio::fs::write(&py_script, args.script.script()).await?;
 
diff --git a/src/script/interpreter/r.rs b/src/script/interpreter/r.rs
index b378ec7c..04ecc798 100644
--- a/src/script/interpreter/r.rs
+++ b/src/script/interpreter/r.rs
@@ -4,13 +4,13 @@ use rattler_conda_types::Platform;
 
 use crate::script::{ExecutionArgs, ResolvedScriptContents};
 
-use super::{BashInterpreter, CmdExeInterpreter, Interpreter, find_interpreter};
+use super::{BashInterpreter, CmdExeInterpreter, Interpreter, InterpreterError, find_interpreter};
 
 pub(crate) struct RInterpreter;
 
 // R interpreter calls either bash or cmd.exe interpreter for activation and then runs R script
 impl Interpreter for RInterpreter {
-    async fn run(&self, args: ExecutionArgs) -> Result<(), std::io::Error> {
+    async fn run(&self, args: ExecutionArgs) -> Result<(), InterpreterError> {
         let script = args.script.script();
         let r_script = args.work_dir.join("conda_build_script.R");
         tokio::fs::write(&r_script, script).await?;
diff --git a/src/script/mod.rs b/src/script/mod.rs
index 4e37f689..82edb800 100644
--- a/src/script/mod.rs
+++ b/src/script/mod.rs
@@ -1,6 +1,7 @@
 //! Module for running scripts in different interpreters.
 mod interpreter;
 mod sandbox;
+pub use interpreter::InterpreterError;
 pub use sandbox::{SandboxArguments, SandboxConfiguration};
 
 use crate::script::interpreter::Interpreter;
@@ -246,7 +247,7 @@ impl Script {
         mut jinja_config: Option<Jinja>,
         sandbox_config: Option<&SandboxConfiguration>,
         debug: Debug,
-    ) -> Result<(), std::io::Error> {
+    ) -> Result<(), InterpreterError> {
         // TODO: This is a bit of an out and about way to determine whether or
         //  not nushell is available. It would be best to run the activation
         //  of the environment and see if nu is on the path, but hat is a
@@ -349,7 +350,7 @@ impl Script {
                     return Err(std::io::Error::new(
                         std::io::ErrorKind::Other,
                         "Nushell is not installed, did you add `nushell` to the build dependencies?".to_string(),
-                    ));
+                    ).into());
                 }
                 NuShellInterpreter.run(exec_args).await?
             }
@@ -362,7 +363,8 @@ impl Script {
                 return Err(std::io::Error::new(
                     std::io::ErrorKind::Other,
                     format!("Unsupported interpreter: {}", interpreter),
-                ));
+                )
+                .into());
             }
         };
 
@@ -437,7 +439,7 @@ impl Output {
     /// - The script file cannot be read or found
     /// - The script execution fails
     /// - The interpreter is not supported or not available
-    pub async fn run_build_script(&self) -> Result<(), std::io::Error> {
+    pub async fn run_build_script(&self) -> Result<(), InterpreterError> {
         let span = tracing::info_span!("Running build script");
         let _enter = span.enter();
 
diff --git a/src/source/url_source.rs b/src/source/url_source.rs
index e4be467c..c3a8400d 100644
--- a/src/source/url_source.rs
+++ b/src/source/url_source.rs
@@ -71,7 +71,7 @@ async fn fetch_remote(
 
     let (mut response, download_size) = {
         let resp = client
-            .get(url.as_str())
+            .get(url.clone())
             .header(reqwest::header::USER_AGENT, APP_USER_AGENT)
             .send()
             .await


Test patch:
diff --git a/src/package_test/run_test.rs b/src/package_test/run_test.rs
index 199bf924..b4a03feb 100644
--- a/src/package_test/run_test.rs
+++ b/src/package_test/run_test.rs
@@ -15,7 +15,6 @@ use std::{
     str::FromStr,
 };
 
-use dunce::canonicalize;
 use fs_err as fs;
 use rattler::package_cache::CacheKey;
 use rattler_conda_types::{
@@ -28,6 +27,7 @@ use rattler_shell::{
     shell::{Shell, ShellEnum},
 };
 use rattler_solve::{ChannelPriority, SolveStrategy};
+use tempfile::TempDir;
 
 use crate::{
     env_vars,
@@ -143,7 +143,7 @@ impl Tests {
                         None,
                         None,
                         None,
-                        Debug::new(true),
+                        Debug::new(false),
                     )
                     .await
                     .map_err(|e| TestError::TestFailed(e.to_string()))?;
@@ -164,7 +164,7 @@ impl Tests {
                         None,
                         None,
                         None,
-                        Debug::new(true),
+                        Debug::new(false),
                     )
                     .await
                     .map_err(|e| TestError::TestFailed(e.to_string()))?;
@@ -220,6 +220,8 @@ pub struct TestConfiguration {
     pub current_platform: PlatformWithVirtualPackages,
     /// If true, the test prefix will not be deleted after the test is run
     pub keep_test_prefix: bool,
+    /// The index of the test to execute. If not set, all tests will be executed.
+    pub test_index: Option<usize>,
     /// The channels to use for the test – do not forget to add the local build
     /// outputs channel if desired
     pub channels: Vec<ChannelUrl>,
@@ -229,6 +231,10 @@ pub struct TestConfiguration {
     pub solve_strategy: SolveStrategy,
     /// The tool configuration
     pub tool_configuration: tool_configuration::Configuration,
+    /// The output directory to create the test prefixes in (will be `output_dir/test`)
+    pub output_dir: PathBuf,
+    /// Debug mode yes, or no
+    pub debug: Debug,
 }
 
 fn env_vars_from_package(index_json: &IndexJson) -> HashMap<String, String> {
@@ -355,10 +361,6 @@ pub async fn run_test(
         fs::remove_dir_all(&package_folder)?;
     }
 
-    let prefix = canonicalize(&config.test_prefix)?;
-
-    tracing::info!("Creating test environment in '{}'", prefix.display());
-
     let mut channels = config.channels.clone();
     channels.insert(0, Channel::from_directory(tmp_repo.path()).base_url);
 
@@ -391,6 +393,11 @@ pub async fn run_test(
     let env = env_vars_from_package(&index_json);
     // extract package in place
     if package_folder.join("info/test").exists() {
+        let prefix =
+            TempDir::with_prefix_in(format!("test_{}", pkg.name), &config.output_dir)?.into_path();
+
+        tracing::info!("Creating test environment in '{}'", prefix.display());
+
         let test_dep_json = PathBuf::from("info/test/test_time_dependencies.json");
         let test_dependencies: Vec<String> = if package_folder.join(&test_dep_json).exists() {
             serde_json::from_str(&fs::read_to_string(package_folder.join(&test_dep_json))?)?
@@ -435,31 +442,57 @@ pub async fn run_test(
             "{} all tests passed!",
             console::style(console::Emoji("✔", "")).green()
         );
+
+        if prefix.exists() {
+            fs::remove_dir_all(prefix)?;
+        }
     }
 
     if package_folder.join("info/tests/tests.yaml").exists() {
         let tests = fs::read_to_string(package_folder.join("info/tests/tests.yaml"))?;
         let tests: Vec<TestType> = serde_yaml::from_str(&tests)?;
 
+        if let Some(test_index) = config.test_index {
+            if test_index >= tests.len() {
+                return Err(TestError::TestFailed(format!(
+                    "Test index {} out of range (0..{})",
+                    test_index,
+                    tests.len()
+                )));
+            }
+        }
+
+        let tests = if let Some(test_index) = config.test_index {
+            vec![tests[test_index].clone()]
+        } else {
+            tests
+        };
+
         for test in tests {
+            let test_prefix =
+                TempDir::with_prefix_in(format!("test_{}", pkg.name), &config.test_prefix)?
+                    .into_path();
             match test {
                 TestType::Command(c) => {
-                    c.run_test(&pkg, &package_folder, &prefix, &config, &env)
+                    c.run_test(&pkg, &package_folder, &test_prefix, &config, &env)
                         .await?
                 }
                 TestType::Python { python } => {
                     python
-                        .run_test(&pkg, &package_folder, &prefix, &config)
+                        .run_test(&pkg, &package_folder, &test_prefix, &config)
                         .await?
                 }
                 TestType::Perl { perl } => {
-                    perl.run_test(&pkg, &package_folder, &prefix, &config)
+                    perl.run_test(&pkg, &package_folder, &test_prefix, &config)
+                        .await?
+                }
+                TestType::R { r } => {
+                    r.run_test(&pkg, &package_folder, &test_prefix, &config)
                         .await?
                 }
-                TestType::R { r } => r.run_test(&pkg, &package_folder, &prefix, &config).await?,
                 TestType::Downstream(downstream) if downstream_package.is_none() => {
                     downstream
-                        .run_test(&pkg, package_file, &prefix, &config)
+                        .run_test(&pkg, package_file, &test_prefix, &config)
                         .await?
                 }
                 TestType::Downstream(_) => {
@@ -470,6 +503,10 @@ pub async fn run_test(
                 // This test already runs during the build process and we don't need to run it again
                 TestType::PackageContents { .. } => {}
             }
+
+            if !config.keep_test_prefix {
+                fs::remove_dir_all(test_prefix)?;
+            }
         }
 
         tracing::info!(
@@ -478,10 +515,6 @@ pub async fn run_test(
         );
     }
 
-    if prefix.exists() {
-        fs::remove_dir_all(prefix)?;
-    }
-
     Ok(())
 }
 
@@ -566,6 +599,7 @@ impl PythonTest {
         let span = tracing::info_span!("", message = %span_message);
         let _guard = span.enter();
 
+        let test_prefix = prefix.join("test_env");
         create_environment(
             "test",
             &dependencies,
@@ -573,7 +607,7 @@ impl PythonTest {
                 .host_platform
                 .as_ref()
                 .unwrap_or(&config.current_platform),
-            prefix,
+            &test_prefix,
             &config.channels,
             &config.tool_configuration,
             config.channel_priority,
@@ -593,17 +627,18 @@ impl PythonTest {
             ..Script::default()
         };
 
-        let tmp_dir = tempfile::tempdir()?;
+        let test_dir = prefix.join("test");
+        fs::create_dir_all(&test_dir)?;
         script
             .run_script(
                 Default::default(),
-                tmp_dir.path(),
+                &test_dir,
                 path,
-                prefix,
+                &test_prefix,
                 None,
                 None,
                 None,
-                Debug::new(true),
+                config.debug,
             )
             .await
             .map_err(|e| TestError::TestFailed(e.to_string()))?;
@@ -623,11 +658,11 @@ impl PythonTest {
                     Default::default(),
                     path,
                     path,
-                    prefix,
+                    &test_prefix,
                     None,
                     None,
                     None,
-                    Debug::new(true),
+                    config.debug,
                 )
                 .await
                 .map_err(|e| TestError::TestFailed(e.to_string()))?;
@@ -660,6 +695,7 @@ impl PerlTest {
 
         let dependencies = vec!["perl".parse().unwrap(), match_spec];
 
+        let test_prefix = prefix.join("test_env");
         create_environment(
             "test",
             &dependencies,
@@ -667,7 +703,7 @@ impl PerlTest {
                 .host_platform
                 .as_ref()
                 .unwrap_or(&config.current_platform),
-            prefix,
+            &test_prefix,
             &config.channels,
             &config.tool_configuration,
             config.channel_priority,
@@ -691,17 +727,18 @@ impl PerlTest {
             ..Script::default()
         };
 
-        let tmp_dir = tempfile::tempdir()?;
+        let test_folder = prefix.join("test_files");
+        fs::create_dir_all(&test_folder)?;
         script
             .run_script(
                 Default::default(),
-                tmp_dir.path(),
+                &test_folder,
                 path,
-                prefix,
+                &test_prefix,
                 None,
                 None,
                 None,
-                Debug::new(true),
+                config.debug,
             )
             .await
             .map_err(|e| TestError::TestFailed(e.to_string()))?;
@@ -727,7 +764,7 @@ impl CommandsTest {
 
         let build_prefix = if !deps.build.is_empty() {
             tracing::info!("Installing build dependencies");
-            let build_prefix = test_directory.join("bld");
+            let build_prefix = test_directory.join("test_build_env");
             let build_dependencies = deps
                 .build
                 .iter()
@@ -768,7 +805,7 @@ impl CommandsTest {
             .as_ref()
             .unwrap_or(&config.current_platform);
 
-        let run_prefix = test_directory.join("run");
+        let run_prefix = test_directory.join("test_run_env");
         create_environment(
             "test",
             &dependencies,
@@ -793,8 +830,8 @@ impl CommandsTest {
 
         // copy all test files to a temporary directory and set it as the working
         // directory
-        let tmp_dir = tempfile::tempdir()?;
-        CopyDir::new(path, tmp_dir.path()).run().map_err(|e| {
+        let test_dir = test_directory.join("test");
+        CopyDir::new(path, &test_dir).run().map_err(|e| {
             TestError::IoError(std::io::Error::new(
                 std::io::ErrorKind::Other,
                 format!("Failed to copy test files: {}", e),
@@ -805,13 +842,13 @@ impl CommandsTest {
         self.script
             .run_script(
                 env_vars,
-                tmp_dir.path(),
+                &test_dir,
                 path,
                 &run_prefix,
                 build_prefix.as_ref(),
                 None,
                 None,
-                Debug::new(true),
+                config.debug,
             )
             .await
             .map_err(|e| TestError::TestFailed(e.to_string()))?;
@@ -925,7 +962,7 @@ impl RTest {
         )?;
 
         let dependencies = vec!["r-base".parse().unwrap(), match_spec];
-
+        let test_prefix = prefix.join("test_env");
         create_environment(
             "test",
             &dependencies,
@@ -933,7 +970,7 @@ impl RTest {
                 .host_platform
                 .as_ref()
                 .unwrap_or(&config.current_platform),
-            prefix,
+            &test_prefix,
             &config.channels,
             &config.tool_configuration,
             config.channel_priority,
@@ -957,17 +994,18 @@ impl RTest {
             ..Script::default()
         };
 
-        let tmp_dir = tempfile::tempdir()?;
+        let test_folder = prefix.join("test_files");
+        fs::create_dir_all(&test_folder)?;
         script
             .run_script(
                 Default::default(),
-                tmp_dir.path(),
+                &test_folder,
                 path,
-                prefix,
+                &test_prefix,
                 None,
                 None,
                 None,
-                Debug::new(true),
+                config.debug,
             )
             .await
             .map_err(|e| TestError::TestFailed(e.to_string()))?;
diff --git a/test-data/recipes/merge_build_and_host/recipe.yaml b/test-data/recipes/merge_build_and_host/recipe.yaml
index 10ebf52c..ca7324b5 100644
--- a/test-data/recipes/merge_build_and_host/recipe.yaml
+++ b/test-data/recipes/merge_build_and_host/recipe.yaml
@@ -34,3 +34,11 @@ build:
     else
       echo "PREFIX and BUILD_PREFIX are the same. Host and build envs have been merged!"
     fi
+
+    # check that we have a file under $PREFIX/conda-meta/history
+    if [[ ! -f ${PREFIX}/conda-meta/history ]]; then
+      echo "history file not found under $PREFIX/conda-meta/history"
+      exit 1
+    else
+      echo "history file found under $PREFIX/conda-meta/history"
+    fi
diff --git a/test/end-to-end/test_simple.py b/test/end-to-end/test_simple.py
index bbd45068..342704fb 100644
--- a/test/end-to-end/test_simple.py
+++ b/test/end-to-end/test_simple.py
@@ -1904,13 +1904,18 @@ def test_merge_build_and_host(
         tmp_path,
     )
 
+
 def test_secret_leaking(rattler_build: RattlerBuild, recipes: Path, tmp_path: Path):
     # build the package with experimental flag to enable the feature
     rattler_build.build(
         recipes / "empty_folder",
         tmp_path,
-        extra_args= ["-c", "https://iamasecretusername:123412341234@foobar.com/some-channel",
-        "-c", "https://bizbar.com/t/token1234567/channel-name"],
+        extra_args=[
+            "-c",
+            "https://iamasecretusername:123412341234@foobar.com/some-channel",
+            "-c",
+            "https://bizbar.com/t/token1234567/channel-name",
+        ],
     )
     pkg = get_extracted_package(tmp_path, "empty_folder")
     # scan all files to make sure that the secret is not present
@@ -1922,4 +1927,3 @@ def test_secret_leaking(rattler_build: RattlerBuild, recipes: Path, tmp_path: Pa
             assert "123412341234" not in content, f"Secret found in {file}"
 
             assert "token1234567" not in content, f"Token found in {file}"
-