Change:
} } <del> // HACK: assume stageN-target, we only want stageN. </del> <ins> // `self.config.stage_id` looks like `stage1-<target_tuplet>`, but we only want // the `stage1` part as that is what the output directories of bootstrap are prefixed with. // Note that this *assumes* build layout from bootstrap is produced as: // // ``` // build/<target_tuplet>/          // <- this is `build_root` // ├── stage0 // ├── stage0-bootstrap-tools // ├── stage0-codegen // ├── stage0-rustc // ├── stage0-std // ├── stage0-sysroot // ├── stage0-tools // ├── stage0-tools-bin // ├── stage1 // ├── stage1-std // ├── stage1-tools // ├── stage1-tools-bin // └── test // ``` // FIXME(jieyouxu): improve the communication between bootstrap and compiletest here so // we don't have to hack out a `stageN`. debug!(?self.config.stage_id); </ins> let stage = self.config.stage_id.split('-').next().unwrap(); <del> // First, we construct the path to the built support library. let mut support_lib_path = PathBuf::new(); support_lib_path.push(&build_root); support_lib_path.push(format!("{}-tools-bin", stage)); support_lib_path.push("librun_make_support.rlib"); let mut stage_std_path = PathBuf::new(); stage_std_path.push(&build_root); stage_std_path.push(&stage); stage_std_path.push("lib"); // Then, we need to build the recipe `rmake.rs` and link in the support library. let recipe_bin = base_dir.join(if self.config.target.contains("windows") { "rmake.exe" } else { "rmake" }); let mut support_lib_deps = PathBuf::new(); support_lib_deps.push(&build_root); support_lib_deps.push(format!("{}-tools", stage)); support_lib_deps.push(&self.config.host); support_lib_deps.push("release"); support_lib_deps.push("deps"); let mut support_lib_deps_deps = PathBuf::new(); support_lib_deps_deps.push(&build_root); support_lib_deps_deps.push(format!("{}-tools", stage)); support_lib_deps_deps.push("release"); support_lib_deps_deps.push("deps"); </del> <ins> // In order to link in the support library as a rlib when compiling recipes, we need three // paths: // 1. Path of the built support library rlib itself. // 2. Path of the built support library's dependencies directory. // 3. Path of the built support library's dependencies' dependencies directory. // // The paths look like // // ``` // build/<target_tuplet>/ // ├── stageN-tools-bin/ // │   └── librun_make_support.rlib       // <- support rlib itself // ├── stageN-tools/ // │   ├── release/deps/                  // <- deps of deps // │   └── <host_tuplet>/release/deps/    // <- deps // ``` // // There almost certainly is a better way to do this, but this seems to work for now. let support_lib_path = { let mut p = build_root.clone(); p.push(format!("{}-tools-bin", stage)); p.push("librun_make_support.rlib"); p }; debug!(?support_lib_path); let support_lib_deps = { let mut p = build_root.clone(); p.push(format!("{}-tools", stage)); p.push(&self.config.host); p.push("release"); p.push("deps"); p }; </ins> debug!(?support_lib_deps); <ins> let support_lib_deps_deps = { let mut p = build_root.clone(); p.push(format!("{}-tools", stage)); p.push("release"); p.push("deps"); p }; </ins> debug!(?support_lib_deps_deps); <del> let orig_dylib_env_paths = </del> <ins> // To compile the recipe with rustc, we need to provide suitable dynamic library search // paths to rustc. This includes both: // 1. The "base" dylib search paths that was provided to compiletest, e.g. `LD_LIBRARY_PATH` //    on some linux distros. // 2. Specific library paths in `self.config.compile_lib_path` needed for running rustc. let base_dylib_search_paths = </ins> Vec::from_iter(env::split_paths(&env::var(dylib_env_var()).unwrap())); <del> let mut host_dylib_env_paths = Vec::new(); host_dylib_env_paths.push(cwd.join(&self.config.compile_lib_path)); host_dylib_env_paths.extend(orig_dylib_env_paths.iter().cloned()); let host_dylib_env_paths = env::join_paths(host_dylib_env_paths).unwrap(); </del> <ins> let host_dylib_search_paths = { let mut paths = vec![self.config.compile_lib_path.clone()]; paths.extend(base_dylib_search_paths.iter().cloned()); paths }; // Calculate the paths of the recipe binary. As previously discussed, this is placed at // `<base_dir>/<bin_name>` with `bin_name` being `rmake` or `rmake.exe` dependending on // platform. let recipe_bin = { let mut p = base_dir.join("rmake"); p.set_extension(env::consts::EXE_EXTENSION); p }; debug!(?recipe_bin); </ins> <del> let mut cmd = Command::new(&self.config.rustc_path); cmd.arg("-o") </del> <ins> // FIXME(jieyouxu): explain what the hecc we are doing here. // FIXME(jieyouxu): audit these env vars. some of them only makes sense for make, not rustc! let mut rustc = Command::new(&self.config.rustc_path); rustc // Specify output path .arg("-o") </ins> .arg(&recipe_bin) <ins> // Specify library search paths for `run_make_support`. </ins> .arg(format!("-Ldependency={}", &support_lib_path.parent().unwrap().to_string_lossy())) .arg(format!("-Ldependency={}", &support_lib_deps.to_string_lossy())) .arg(format!("-Ldependency={}", &support_lib_deps_deps.to_string_lossy())) <ins> // Provide `run_make_support` as extern prelude, so test writers don't need to write // `extern run_make_support;`. </ins> .arg("--extern") .arg(format!("run_make_support={}", &support_lib_path.to_string_lossy())) <ins> // Default to Edition 2021. </ins> .arg("--edition=2021") <ins> // The recipe file itself. </ins> .arg(&self.testpaths.file.join("rmake.rs")) <del> .env("TARGET", &self.config.target) .env("PYTHON", &self.config.python) .env("RUST_BUILD_STAGE", &self.config.stage_id) .env("RUSTC", cwd.join(&self.config.rustc_path)) .env("LD_LIB_PATH_ENVVAR", dylib_env_var()) .env(dylib_env_var(), &host_dylib_env_paths) .env("HOST_RPATH_DIR", cwd.join(&self.config.compile_lib_path)) .env("TARGET_RPATH_DIR", cwd.join(&self.config.run_lib_path)) .env("LLVM_COMPONENTS", &self.config.llvm_components); </del> <ins> // Provide necessary library search paths for rustc. .env(dylib_env_var(), &env::join_paths(host_dylib_search_paths).unwrap()); </ins> // In test code we want to be very pedantic about values being silently discarded that are // annotated with `#[must_use]`. <del> cmd.arg("-Dunused_must_use"); </del> <ins> rustc.arg("-Dunused_must_use"); // FIXME(jieyouxu): explain this! </ins> if std::env::var_os("COMPILETEST_FORCE_STAGE0").is_some() { let mut stage0_sysroot = build_root.clone(); stage0_sysroot.push("stage0-sysroot"); debug!(?stage0_sysroot); debug!(exists = stage0_sysroot.exists()); <del> cmd.arg("--sysroot").arg(&stage0_sysroot); </del> <ins> rustc.arg("--sysroot").arg(&stage0_sysroot); </ins> } <del> let res = self.run_command_to_procres(&mut cmd); </del> <ins> let res = self.run_command_to_procres(&mut rustc); </ins> if !res.status.success() { self.fatal_proc_rec("run-make test failed: could not build `rmake.rs` recipe", &res); } <del> // Finally, we need to run the recipe binary to build and run the actual tests. debug!(?recipe_bin); </del> <ins> // FIXME(jieyouxu): explain what the hecc we are doing here. let mut stage_std_path = PathBuf::new(); stage_std_path.push(&build_root); stage_std_path.push(&stage); stage_std_path.push("lib"); </ins> <del> let mut dylib_env_paths = orig_dylib_env_paths.clone(); dylib_env_paths.push(support_lib_path.parent().unwrap().to_path_buf()); dylib_env_paths.push(stage_std_path.join("rustlib").join(&self.config.host).join("lib")); let dylib_env_paths = env::join_paths(dylib_env_paths).unwrap(); </del> <ins> // FIXME(jieyouxu): explain what the hecc we are doing here. let recipe_dylib_search_paths = { let mut paths = base_dylib_search_paths.clone(); paths.push(support_lib_path.parent().unwrap().to_path_buf()); paths.push(stage_std_path.join("rustlib").join(&self.config.host).join("lib")); paths }; </ins> <del> let mut target_rpath_env_path = Vec::new(); target_rpath_env_path.push(&rmake_out_dir); target_rpath_env_path.extend(&orig_dylib_env_paths); let target_rpath_env_path = env::join_paths(target_rpath_env_path).unwrap(); </del> <ins> // FIXME(jieyouxu): explain what the hecc we are doing here. let target_rpaths = { let mut paths = vec![rmake_out_dir.clone()]; paths.extend(base_dylib_search_paths.iter().cloned()); paths }; // FIXME(jieyouxu): explain what the hecc we are doing here. // FIXME(jieyouxu): audit these env vars. some of them only makes sense for make, not rustc! </ins> let mut cmd = Command::new(&recipe_bin); cmd.current_dir(&rmake_out_dir) .stdout(Stdio::piped()) .stderr(Stdio::piped()) .env("LD_LIB_PATH_ENVVAR", dylib_env_var()) <del> .env("TARGET_RPATH_ENV", &target_rpath_env_path) .env(dylib_env_var(), &dylib_env_paths) </del> <ins> .env("TARGET_RPATH_ENV", &env::join_paths(target_rpaths).unwrap()) .env(dylib_env_var(), &env::join_paths(recipe_dylib_search_paths).unwrap()) </ins> .env("TARGET", &self.config.target) .env("PYTHON", &self.config.python) <del> .env("SOURCE_ROOT", &src_root) </del> <ins> .env("SOURCE_ROOT", &source_root) </ins> .env("RUST_BUILD_STAGE", &self.config.stage_id) <del> .env("RUSTC", cwd.join(&self.config.rustc_path)) .env("HOST_RPATH_DIR", cwd.join(&self.config.compile_lib_path)) .env("TARGET_RPATH_DIR", cwd.join(&self.config.run_lib_path)) </del> <ins> .env("RUSTC", &self.config.rustc_path) .env("HOST_RPATH_DIR", &self.config.compile_lib_path) .env("TARGET_RPATH_DIR", &self.config.run_lib_path) </ins> .env("LLVM_COMPONENTS", &self.config.llvm_components); if let Some(ref rustdoc) = self.config.rustdoc_path { <del> cmd.env("RUSTDOC", cwd.join(rustdoc)); </del> <ins> cmd.env("RUSTDOC", source_root.join(rustdoc)); </ins> } if let Some(ref node) = self.config.nodejs {

Discussion:
I've noticed when trying to debug why certain PRs were failing that our setup in compiletest contains "dead" env vars and some other hacks that warrant revisiting for reliability and correctness. I should revise how tests are setup in compiletest.