Repository: cross-rs/cross
Pull request #1085 (2022-10-19 00:33:04)

Fixes custom toolchain support without rustup.
Avoids checking for installed toolchains, and also prevents renaming the sysroot based on the toolchain full name, since this may be in a directory completely different than the target name.

Alexhuszagh: @otavio Can you test if this works for you? It only checks if the toolchain is installed if not using a custom toolchain, but also avoids naming the sysroot in that case. This means that the above example where we had a sysroot of `/usr` being renamed to `/stable-x86_64-unknown-linux-gnu` won't happen with custom toolchains.

This should supersede #1083.
otavio: It worked just fine.
Emilgardis: bors try --none

just want some artifacts :)
Emilgardis: bors try-

bors try --target x86_64-unknown-linux-gnu x86_64-pc-windows-msvc
bors[bot]: ## try

Build succeeded:
  * [conclusion](https://github.com/cross-rs/cross/actions/runs/3278784626/jobs/5397682808)
Emilgardis: I think we can improve this story a bit.

When using a non-rustup toolchain installed by e.g a package manager, you'll get the following error message


```
emil@debian-gnu-linux-10:/home/emil/dev_space/aaaaaa-2018$ CROSS_CUSTOM=1 cross build
Error: 
   0: toolchain is not fully qualified

Location:
   src/rustc.rs:268

Backtrace omitted. Run with RUST_BACKTRACE=1 environment variable to display it.
Run with RUST_BACKTRACE=full to include source snippets.
```

we could add a `with_suggestion` here, that suggestion could link to a wiki entry, or just say `set CROSS_CUSTOM_TOOLCHAIN=1 or install rust via rustup`
Emilgardis: We already have a similar message here, but that only works if the sysroot looks like a rustc triple

https://github.com/Emilgardis/cross/blob/2a210f69e6f43ec781c96e6e1a0d3ed18c643d96/src/rustup.rs#L97
Alexhuszagh: I've changed the error message to be:

```rust
Ok(_) => return Err(eyre::eyre!("toolchain is not fully qualified")
    .with_suggestion(|| {
        "if using a custom toolchain try set `CROSS_CUSTOM_TOOLCHAIN=1` or install rust via rustup"
})),
```

Patch:
diff --git a/.changes/1084.json b/.changes/1084.json
deleted file mode 100644
index 39d80ea..0000000
--- a/.changes/1084.json
+++ /dev/null
@@ -1,5 +0,0 @@
-{
-    "type": "fixed",
-    "description": "fixed remote docker data volume paths on windows",
-    "issues": [1081]
-}
diff --git a/.changes/1085.json b/.changes/1085.json
new file mode 100644
index 0000000..373c306
--- /dev/null
+++ b/.changes/1085.json
@@ -0,0 +1,4 @@
+{
+    "type": "added",
+    "description": "support custom toolchains without rustup."
+}
diff --git a/docker/Dockerfile.aarch64-linux-android b/docker/Dockerfile.aarch64-linux-android
index 412fe0d..5ccd19a 100644
--- a/docker/Dockerfile.aarch64-linux-android
+++ b/docker/Dockerfile.aarch64-linux-android
@@ -31,7 +31,7 @@ ENV CARGO_TARGET_AARCH64_LINUX_ANDROID_LINKER=aarch64-linux-android-gcc \
     CARGO_TARGET_AARCH64_LINUX_ANDROID_RUNNER="/android-runner aarch64" \
     CC_aarch64_linux_android=aarch64-linux-android-gcc \
     CXX_aarch64_linux_android=aarch64-linux-android-g++ \
-    BINDGEN_EXTRA_CLANG_ARGS_aarch64_linux_android="--sysroot=$CROSS_SYSROOT" \
+    BINDGEN_EXTRA_CLANG_ARGS_aarch64_linux_android="--sysroot=$CROSS_SYSROOTt" \
     DEP_Z_INCLUDE="$CROSS_SYSROOT/usr/include"/ \
     RUST_TEST_THREADS=1 \
     HOME=/tmp/ \
diff --git a/src/bin/commands/containers.rs b/src/bin/commands/containers.rs
index 65f121c..1d6e226 100644
--- a/src/bin/commands/containers.rs
+++ b/src/bin/commands/containers.rs
@@ -445,9 +445,9 @@ pub fn create_persistent_volume(
     docker.run_and_get_status(msg_info, true)?;
 
     let data_volume = docker::ContainerDataVolume::new(engine, &container_id, &dirs);
-    data_volume.copy_xargo(mount_prefix, msg_info)?;
-    data_volume.copy_cargo(mount_prefix, copy_registry, msg_info)?;
-    data_volume.copy_rust(None, mount_prefix, msg_info)?;
+    data_volume.copy_xargo(mount_prefix.as_ref(), msg_info)?;
+    data_volume.copy_cargo(mount_prefix.as_ref(), copy_registry, msg_info)?;
+    data_volume.copy_rust(None, mount_prefix.as_ref(), msg_info)?;
 
     docker::ChildContainer::finish_static(is_tty, msg_info);
 
diff --git a/src/docker/remote.rs b/src/docker/remote.rs
index 8f4dede..cba7565 100644
--- a/src/docker/remote.rs
+++ b/src/docker/remote.rs
@@ -1,6 +1,6 @@
 use std::collections::BTreeMap;
 use std::io::{self, BufRead, Read, Write};
-use std::path::Path;
+use std::path::{Path, PathBuf};
 use std::process::{Command, ExitStatus};
 use std::{env, fs, time};
 
@@ -34,40 +34,25 @@ fn subcommand_or_exit(engine: &Engine, cmd: &str) -> Result<Command> {
     Ok(engine.subcommand(cmd))
 }
 
-pub fn posix_parent(path: &str) -> Option<&str> {
-    Path::new(path).parent()?.to_str()
-}
-
 impl<'a, 'b, 'c> ContainerDataVolume<'a, 'b, 'c> {
-    // NOTE: `reldir` should be a relative POSIX path to the root directory
-    // on windows, this should be something like `mnt/c`. that is, all paths
-    // inside the container should not have the mount prefix.
-    fn create_dir(
-        &self,
-        reldir: &str,
-        mount_prefix: &str,
-        msg_info: &mut MessageInfo,
-    ) -> Result<ExitStatus> {
+    fn create_dir(&self, dir: &Path, msg_info: &mut MessageInfo) -> Result<ExitStatus> {
         // make our parent directory if needed
         subcommand_or_exit(self.engine, "exec")?
             .arg(self.container)
-            .args(["sh", "-c", &format!("mkdir -p '{mount_prefix}/{reldir}'")])
+            .args([
+                "sh",
+                "-c",
+                &format!("mkdir -p '{}'", dir.as_posix_absolute()?),
+            ])
             .run_and_get_status(msg_info, false)
     }
 
     // copy files for a docker volume, for remote host support
-    // NOTE: `reldst` has the same caveats as `reldir` in `create_dir`.
-    fn copy_files(
-        &self,
-        src: &Path,
-        reldst: &str,
-        mount_prefix: &str,
-        msg_info: &mut MessageInfo,
-    ) -> Result<ExitStatus> {
+    fn copy_files(&self, src: &Path, dst: &Path, msg_info: &mut MessageInfo) -> Result<ExitStatus> {
         subcommand_or_exit(self.engine, "cp")?
             .arg("-a")
             .arg(src.to_utf8()?)
-            .arg(format!("{}:{mount_prefix}/{reldst}", self.container))
+            .arg(format!("{}:{}", self.container, dst.as_posix_absolute()?))
             .run_and_get_status(msg_info, false)
     }
 
@@ -75,8 +60,7 @@ impl<'a, 'b, 'c> ContainerDataVolume<'a, 'b, 'c> {
     fn copy_files_nocache(
         &self,
         src: &Path,
-        reldst: &str,
-        mount_prefix: &str,
+        dst: &Path,
         copy_symlinks: bool,
         msg_info: &mut MessageInfo,
     ) -> Result<ExitStatus> {
@@ -87,7 +71,7 @@ impl<'a, 'b, 'c> ContainerDataVolume<'a, 'b, 'c> {
         let temppath = tempdir.path();
         let had_symlinks = copy_dir(src, temppath, copy_symlinks, 0, |e, _| is_cachedir(e))?;
         warn_symlinks(had_symlinks, msg_info)?;
-        self.copy_files(&temppath.join("."), reldst, mount_prefix, msg_info)
+        self.copy_files(&temppath.join("."), dst, msg_info)
     }
 
     // copy files for a docker volume, for remote host support
@@ -95,8 +79,7 @@ impl<'a, 'b, 'c> ContainerDataVolume<'a, 'b, 'c> {
     fn copy_file_list(
         &self,
         src: &Path,
-        reldst: &str,
-        mount_prefix: &str,
+        dst: &Path,
         files: &[&str],
         msg_info: &mut MessageInfo,
     ) -> Result<ExitStatus> {
@@ -111,15 +94,14 @@ impl<'a, 'b, 'c> ContainerDataVolume<'a, 'b, 'c> {
         }
         // if copying from a src directory to dst directory with docker, to
         // copy the contents from `src` into `dst`, `src` must end with `/.`
-        self.copy_files(&temppath.join("."), reldst, mount_prefix, msg_info)
+        self.copy_files(&temppath.join("."), dst, msg_info)
     }
 
     // removed files from a docker volume, for remote host support
     // provides a list of files relative to src.
     fn remove_file_list(
         &self,
-        reldst: &str,
-        mount_prefix: &str,
+        dst: &Path,
         files: &[&str],
         msg_info: &mut MessageInfo,
     ) -> Result<ExitStatus> {
@@ -140,7 +122,7 @@ impl<'a, 'b, 'c> ContainerDataVolume<'a, 'b, 'c> {
         // SAFETY: safe, single-threaded execution.
         let mut tempfile = unsafe { temp::TempFile::new()? };
         for file in files {
-            writeln!(tempfile.file(), "{mount_prefix}/{reldst}/{file}")?;
+            writeln!(tempfile.file(), "{}", dst.join(file).as_posix_relative()?)?;
         }
 
         // need to avoid having hundreds of files on the command, so
@@ -156,39 +138,27 @@ impl<'a, 'b, 'c> ContainerDataVolume<'a, 'b, 'c> {
             .run_and_get_status(msg_info, true)
     }
 
-    fn container_path_exists(
-        &self,
-        relpath: &str,
-        mount_prefix: &str,
-        msg_info: &mut MessageInfo,
-    ) -> Result<bool> {
+    fn container_path_exists(&self, path: &Path, msg_info: &mut MessageInfo) -> Result<bool> {
         Ok(subcommand_or_exit(self.engine, "exec")?
             .arg(self.container)
             .args([
                 "bash",
                 "-c",
-                &format!("[[ -d '{mount_prefix}/{relpath}' ]]"),
+                &format!("[[ -d '{}' ]]", path.as_posix_absolute()?),
             ])
             .run_and_get_status(msg_info, true)?
             .success())
     }
 
-    pub fn copy_xargo(&self, mount_prefix: &str, msg_info: &mut MessageInfo) -> Result<()> {
+    pub fn copy_xargo(&self, mount_prefix: &Path, msg_info: &mut MessageInfo) -> Result<()> {
         let dirs = &self.toolchain_dirs;
-        let reldst = dirs.xargo_mount_path_relative()?;
+        let dst = mount_prefix.join(&dirs.xargo_mount_path_relative()?);
         if dirs.xargo().exists() {
             self.create_dir(
-                // this always works, even if we have `/xargo`, since
-                // this will be an absolute path. passing an empty path
-                // to `create_dir` isn't an issue.
-                posix_parent(dirs.xargo_mount_path())
-                    .expect("destination should have a parent")
-                    .strip_prefix('/')
-                    .expect("parent directory must be absolute"),
-                mount_prefix,
+                dst.parent().expect("destination should have a parent"),
                 msg_info,
             )?;
-            self.copy_files(dirs.xargo(), &reldst, mount_prefix, msg_info)?;
+            self.copy_files(dirs.xargo(), &dst, msg_info)?;
         }
 
         Ok(())
@@ -196,21 +166,21 @@ impl<'a, 'b, 'c> ContainerDataVolume<'a, 'b, 'c> {
 
     pub fn copy_cargo(
         &self,
-        mount_prefix: &str,
+        mount_prefix: &Path,
         copy_registry: bool,
         msg_info: &mut MessageInfo,
     ) -> Result<()> {
         let dirs = &self.toolchain_dirs;
-        let reldst = dirs.cargo_mount_path_relative()?;
+        let dst = mount_prefix.join(&dirs.cargo_mount_path_relative()?);
         let copy_registry = env::var("CROSS_REMOTE_COPY_REGISTRY")
             .map(|s| bool_from_envvar(&s))
             .unwrap_or(copy_registry);
 
         if copy_registry {
-            self.copy_files(dirs.cargo(), &reldst, mount_prefix, msg_info)?;
+            self.copy_files(dirs.cargo(), &dst, msg_info)?;
         } else {
             // can copy a limit subset of files: the rest is present.
-            self.create_dir(&reldst, mount_prefix, msg_info)?;
+            self.create_dir(&dst, msg_info)?;
             for entry in fs::read_dir(dirs.cargo())
                 .wrap_err_with(|| format!("when reading directory {:?}", dirs.cargo()))?
             {
@@ -221,7 +191,7 @@ impl<'a, 'b, 'c> ContainerDataVolume<'a, 'b, 'c> {
                     .wrap_err_with(|| format!("when reading file {file:?}"))?
                     .to_owned();
                 if !basename.starts_with('.') && !matches!(basename.as_ref(), "git" | "registry") {
-                    self.copy_files(&file.path(), &reldst, mount_prefix, msg_info)?;
+                    self.copy_files(&file.path(), &dst, msg_info)?;
                 }
             }
         }
@@ -230,17 +200,17 @@ impl<'a, 'b, 'c> ContainerDataVolume<'a, 'b, 'c> {
     }
 
     // copy over files needed for all targets in the toolchain that should never change
-    fn copy_rust_base(&self, mount_prefix: &str, msg_info: &mut MessageInfo) -> Result<()> {
+    fn copy_rust_base(&self, mount_prefix: &Path, msg_info: &mut MessageInfo) -> Result<()> {
         let dirs = &self.toolchain_dirs;
 
         // the rust toolchain is quite large, but most of it isn't needed
         // we need the bin, libexec, and etc directories, and part of the lib directory.
-        let reldst = dirs.sysroot_mount_path_relative()?;
-        let rustlib = "lib/rustlib";
-        self.create_dir(&format!("{reldst}/{}", rustlib), mount_prefix, msg_info)?;
+        let dst = mount_prefix.join(&dirs.sysroot_mount_path_relative()?);
+        let rustlib = Path::new("lib").join("rustlib");
+        self.create_dir(&dst.join(&rustlib), msg_info)?;
         for basename in ["bin", "libexec", "etc"] {
             let file = dirs.get_sysroot().join(basename);
-            self.copy_files(&file, &reldst, mount_prefix, msg_info)?;
+            self.copy_files(&file, &dst, msg_info)?;
         }
 
         // the lib directories are rather large, so we want only a subset.
@@ -251,7 +221,7 @@ impl<'a, 'b, 'c> ContainerDataVolume<'a, 'b, 'c> {
         // SAFETY: safe, single-threaded execution.
         let tempdir = unsafe { temp::TempDir::new()? };
         let temppath = tempdir.path();
-        file::create_dir_all(&temppath.join(rustlib))?;
+        file::create_dir_all(&temppath.join(&rustlib))?;
         let mut had_symlinks = copy_dir(
             &dirs.get_sysroot().join("lib"),
             &temppath.join("lib"),
@@ -262,37 +232,37 @@ impl<'a, 'b, 'c> ContainerDataVolume<'a, 'b, 'c> {
 
         // next, copy the src/etc directories inside rustlib
         had_symlinks |= copy_dir(
-            &dirs.get_sysroot().join(rustlib),
-            &temppath.join(rustlib),
+            &dirs.get_sysroot().join(&rustlib),
+            &temppath.join(&rustlib),
             true,
             0,
             |e, d| d == 0 && !(e.file_name() == "src" || e.file_name() == "etc"),
         )?;
-        self.copy_files(&temppath.join("lib"), &reldst, mount_prefix, msg_info)?;
+        self.copy_files(&temppath.join("lib"), &dst, msg_info)?;
 
         warn_symlinks(had_symlinks, msg_info)
     }
 
-    fn copy_rust_manifest(&self, mount_prefix: &str, msg_info: &mut MessageInfo) -> Result<()> {
+    fn copy_rust_manifest(&self, mount_prefix: &Path, msg_info: &mut MessageInfo) -> Result<()> {
         let dirs = &self.toolchain_dirs;
 
         // copy over all the manifest files in rustlib
         // these are small text files containing names/paths to toolchains
-        let reldst = dirs.sysroot_mount_path_relative()?;
-        let rustlib = "lib/rustlib";
+        let dst = mount_prefix.join(&dirs.sysroot_mount_path_relative()?);
+        let rustlib = Path::new("lib").join("rustlib");
 
         // SAFETY: safe, single-threaded execution.
         let tempdir = unsafe { temp::TempDir::new()? };
         let temppath = tempdir.path();
-        file::create_dir_all(&temppath.join(rustlib))?;
+        file::create_dir_all(&temppath.join(&rustlib))?;
         let had_symlinks = copy_dir(
-            &dirs.get_sysroot().join(rustlib),
-            &temppath.join(rustlib),
+            &dirs.get_sysroot().join(&rustlib),
+            &temppath.join(&rustlib),
             true,
             0,
             |e, d| d != 0 || e.file_type().map(|t| !t.is_file()).unwrap_or(true),
         )?;
-        self.copy_files(&temppath.join("lib"), &reldst, mount_prefix, msg_info)?;
+        self.copy_files(&temppath.join("lib"), &dst, msg_info)?;
 
         warn_symlinks(had_symlinks, msg_info)
     }
@@ -301,30 +271,30 @@ impl<'a, 'b, 'c> ContainerDataVolume<'a, 'b, 'c> {
     fn copy_rust_triple(
         &self,
         target_triple: &TargetTriple,
-        mount_prefix: &str,
+        mount_prefix: &Path,
         skip_exists: bool,
         msg_info: &mut MessageInfo,
     ) -> Result<()> {
         let dirs = &self.toolchain_dirs;
 
         // copy over the files for a specific triple
-        let reldst = &dirs.sysroot_mount_path_relative()?;
-        let rustlib = "lib/rustlib";
-        let reldst_rustlib = format!("{reldst}/{rustlib}");
+        let dst = mount_prefix.join(&dirs.sysroot_mount_path_relative()?);
+        let rustlib = Path::new("lib").join("rustlib");
+        let dst_rustlib = dst.join(&rustlib);
         let src_toolchain = dirs
             .get_sysroot()
-            .join(Path::new(rustlib))
+            .join(&rustlib)
             .join(target_triple.triple());
-        let reldst_toolchain = format!("{reldst_rustlib}/{}", target_triple.triple());
+        let dst_toolchain = dst_rustlib.join(target_triple.triple());
 
         // skip if the toolchain target component already exists. for the host toolchain
         // or the first run of the target toolchain, we know it doesn't exist.
         let mut skip = false;
         if skip_exists {
-            skip = self.container_path_exists(&reldst_toolchain, mount_prefix, msg_info)?;
+            skip = self.container_path_exists(&dst_toolchain, msg_info)?;
         }
         if !skip {
-            self.copy_files(&src_toolchain, &reldst_rustlib, mount_prefix, msg_info)?;
+            self.copy_files(&src_toolchain, &dst_rustlib, msg_info)?;
         }
         if !skip && skip_exists {
             // this means we have a persistent data volume and we have a
@@ -338,7 +308,7 @@ impl<'a, 'b, 'c> ContainerDataVolume<'a, 'b, 'c> {
     pub fn copy_rust(
         &self,
         target_triple: Option<&TargetTriple>,
-        mount_prefix: &str,
+        mount_prefix: &Path,
         msg_info: &mut MessageInfo,
     ) -> Result<()> {
         let dirs = &self.toolchain_dirs;
@@ -358,17 +328,16 @@ impl<'a, 'b, 'c> ContainerDataVolume<'a, 'b, 'c> {
     fn copy_mount(
         &self,
         src: &Path,
-        reldst: &str,
-        mount_prefix: &str,
+        dst: &Path,
         volume: &VolumeId,
         copy_cache: bool,
         msg_info: &mut MessageInfo,
     ) -> Result<()> {
         let copy_all = |info: &mut MessageInfo| {
             if copy_cache {
-                self.copy_files(src, reldst, mount_prefix, info)
+                self.copy_files(src, dst, info)
             } else {
-                self.copy_files_nocache(src, reldst, mount_prefix, true, info)
+                self.copy_files_nocache(src, dst, true, info)
             }
         };
         match volume {
@@ -382,16 +351,14 @@ impl<'a, 'b, 'c> ContainerDataVolume<'a, 'b, 'c> {
                 let current = Fingerprint::read_dir(src, copy_cache)?;
                 // need to check if the container path exists, otherwise we might
                 // have stale data: the persistent volume was deleted & recreated.
-                if fingerprint.exists()
-                    && self.container_path_exists(reldst, mount_prefix, msg_info)?
-                {
+                if fingerprint.exists() && self.container_path_exists(dst, msg_info)? {
                     let previous = Fingerprint::read_file(&fingerprint)?;
                     let (to_copy, to_remove) = previous.difference(&current);
                     if !to_copy.is_empty() {
-                        self.copy_file_list(src, reldst, mount_prefix, &to_copy, msg_info)?;
+                        self.copy_file_list(src, dst, &to_copy, msg_info)?;
                     }
                     if !to_remove.is_empty() {
-                        self.remove_file_list(reldst, mount_prefix, &to_remove, msg_info)?;
+                        self.remove_file_list(dst, &to_remove, msg_info)?;
                     }
 
                     // write fingerprint afterwards, in case any failure so we
@@ -765,23 +732,24 @@ pub(crate) fn run(
     let copy_cache = env::var("CROSS_REMOTE_COPY_CACHE")
         .map(|s| bool_from_envvar(&s))
         .unwrap_or_default();
-    let copy = |src, reldst: &str, info: &mut MessageInfo| {
-        data_volume.copy_mount(src, reldst, mount_prefix, &volume, copy_cache, info)
+    let copy = |src, dst: &PathBuf, info: &mut MessageInfo| {
+        data_volume.copy_mount(src, dst, &volume, copy_cache, info)
     };
+    let mount_prefix_path = mount_prefix.as_ref();
     if let VolumeId::Discard = volume {
         data_volume
-            .copy_xargo(mount_prefix, msg_info)
+            .copy_xargo(mount_prefix_path, msg_info)
             .wrap_err("when copying xargo")?;
         data_volume
-            .copy_cargo(mount_prefix, false, msg_info)
+            .copy_cargo(mount_prefix_path, false, msg_info)
             .wrap_err("when copying cargo")?;
         data_volume
-            .copy_rust(Some(target.target()), mount_prefix, msg_info)
+            .copy_rust(Some(target.target()), mount_prefix_path, msg_info)
             .wrap_err("when copying rust")?;
     } else {
         // need to copy over the target triple if it hasn't been previously copied
         data_volume
-            .copy_rust_triple(target.target(), mount_prefix, true, msg_info)
+            .copy_rust_triple(target.target(), mount_prefix_path, true, msg_info)
             .wrap_err("when copying rust target files")?;
     }
     // cannot panic: absolute unix path, must have root
@@ -789,41 +757,46 @@ pub(crate) fn run(
         .mount_root()
         .strip_prefix('/')
         .expect("mount root should be absolute");
+    let mount_root = mount_prefix_path.join(rel_mount_root);
     if !rel_mount_root.is_empty() {
         data_volume
             .create_dir(
-                posix_parent(rel_mount_root).expect("mount root should have a parent directory"),
-                mount_prefix,
+                mount_root
+                    .parent()
+                    .expect("mount root should have a parent directory"),
                 msg_info,
             )
             .wrap_err("when creating mount root")?;
     }
-    copy(package_dirs.host_root(), rel_mount_root, msg_info).wrap_err("when copying project")?;
+    copy(package_dirs.host_root(), &mount_root, msg_info).wrap_err("when copying project")?;
     let sysroot = toolchain_dirs.get_sysroot().to_owned();
     let mut copied = vec![
         (
             toolchain_dirs.xargo(),
-            toolchain_dirs.xargo_mount_path_relative()?,
+            mount_prefix_path.join(&toolchain_dirs.xargo_mount_path_relative()?),
         ),
         (
             toolchain_dirs.cargo(),
-            toolchain_dirs.cargo_mount_path_relative()?,
+            mount_prefix_path.join(&toolchain_dirs.cargo_mount_path_relative()?),
+        ),
+        (
+            &sysroot,
+            mount_prefix_path.join(&toolchain_dirs.sysroot_mount_path_relative()?),
         ),
-        (&sysroot, toolchain_dirs.sysroot_mount_path_relative()?),
-        (package_dirs.host_root(), rel_mount_root.to_owned()),
+        (package_dirs.host_root(), mount_root.clone()),
     ];
     let mut to_symlink = vec![];
     let target_dir = file::canonicalize(package_dirs.target())?;
     let target_dir = if let Ok(relpath) = target_dir.strip_prefix(package_dirs.host_root()) {
-        relpath.as_posix_relative()?
+        mount_root.join(relpath)
     } else {
         // outside project, need to copy the target data over
         // only do if we're copying over cached files.
-        let target_dir = "target".to_owned();
+        let target_dir = mount_prefix_path.join("target");
         if copy_cache {
             copy(package_dirs.target(), &target_dir, msg_info)?;
         } else {
-            data_volume.create_dir(&target_dir, mount_prefix, msg_info)?;
+            data_volume.create_dir(&target_dir, msg_info)?;
         }
 
         copied.push((package_dirs.target(), target_dir.clone()));
@@ -835,21 +808,22 @@ pub(crate) fn run(
             // path has already been copied over
             let relpath = src
                 .strip_prefix(psrc)
-                .expect("source should start with prefix")
-                .as_posix_relative()?;
-            to_symlink.push((format!("{pdst}/{relpath}"), dst));
+                .expect("source should start with prefix");
+            to_symlink.push((pdst.join(relpath), dst));
         } else {
-            let reldst = dst
+            let rel_dst = dst
                 .strip_prefix('/')
                 .expect("destination should be absolute");
-            if !reldst.is_empty() {
+            let mount_dst = mount_prefix_path.join(rel_dst);
+            if !rel_dst.is_empty() {
                 data_volume.create_dir(
-                    posix_parent(reldst).expect("destination should have a parent directory"),
-                    mount_prefix,
+                    mount_dst
+                        .parent()
+                        .expect("destination should have a parent directory"),
                     msg_info,
                 )?;
             }
-            copy(src, reldst, msg_info)?;
+            copy(src, &mount_dst, msg_info)?;
         }
     }
 
@@ -859,17 +833,18 @@ pub(crate) fn run(
     let mut final_args = vec![];
     let mut iter = args.iter().cloned();
     let mut has_target_dir = false;
+    let target_dir_string = target_dir.as_posix_absolute()?;
     while let Some(arg) = iter.next() {
         if arg == "--target-dir" {
             has_target_dir = true;
             final_args.push(arg);
             if iter.next().is_some() {
-                final_args.push(target_dir.clone());
+                final_args.push(target_dir_string.clone());
             }
         } else if arg.starts_with("--target-dir=") {
             has_target_dir = true;
             if arg.split_once('=').is_some() {
-                final_args.push(format!("--target-dir={target_dir}"));
+                final_args.push(format!("--target-dir={target_dir_string}"));
             }
         } else {
             final_args.push(arg);
@@ -877,7 +852,7 @@ pub(crate) fn run(
     }
     if !has_target_dir {
         final_args.push("--target-dir".to_owned());
-        final_args.push(target_dir.clone());
+        final_args.push(target_dir_string);
     }
     let mut cmd = options.cargo_variant.safe_command();
     cmd.args(final_args);
@@ -914,7 +889,11 @@ symlink_recurse \"${{prefix}}\"
 "
     ));
     for (src, dst) in to_symlink {
-        symlink.push(format!("ln -s \"{src}\" \"{dst}\"",));
+        symlink.push(format!(
+            "ln -s \"{}\" \"{}\"",
+            src.as_posix_absolute()?,
+            dst
+        ));
     }
     subcommand_or_exit(engine, "exec")?
         .arg(&container_id)
@@ -940,10 +919,13 @@ symlink_recurse \"${{prefix}}\"
         .map(|s| bool_from_envvar(&s))
         .unwrap_or_default();
     bail_container_exited!();
-    if !skip_artifacts && data_volume.container_path_exists(&target_dir, mount_prefix, msg_info)? {
+    if !skip_artifacts && data_volume.container_path_exists(&target_dir, msg_info)? {
         subcommand_or_exit(engine, "cp")?
             .arg("-a")
-            .arg(&format!("{container_id}:{target_dir}",))
+            .arg(&format!(
+                "{container_id}:{}",
+                target_dir.as_posix_absolute()?
+            ))
             .arg(
                 package_dirs
                     .target()
diff --git a/src/docker/shared.rs b/src/docker/shared.rs
index e4874ee..13ab39f 100644
--- a/src/docker/shared.rs
+++ b/src/docker/shared.rs
@@ -1625,6 +1625,7 @@ mod tests {
                 &None,
                 &image_platform,
                 &sysroot,
+                false,
             ))
         }
 
diff --git a/src/lib.rs b/src/lib.rs
index 363b013..5233fe0 100644
--- a/src/lib.rs
+++ b/src/lib.rs
@@ -578,11 +578,10 @@ To override the toolchain mounted in the image, set `target.{}.image.toolchain =
             // set the sysroot explicitly to the toolchain
             let mut is_nightly = toolchain.channel.contains("nightly");
 
-            let installed_toolchains = rustup::installed_toolchains(msg_info)?;
-
-            if !installed_toolchains
-                .into_iter()
-                .any(|t| t == toolchain.to_string())
+            if !toolchain.is_custom
+                && !rustup::installed_toolchains(msg_info)?
+                    .into_iter()
+                    .any(|t| t == toolchain.to_string())
             {
                 rustup::install_toolchain(&toolchain, msg_info)?;
             }
diff --git a/src/rustc.rs b/src/rustc.rs
index 97a61a1..86dc99b 100644
--- a/src/rustc.rs
+++ b/src/rustc.rs
@@ -92,12 +92,18 @@ pub struct QualifiedToolchain {
 }
 
 impl QualifiedToolchain {
-    pub fn new(channel: &str, date: &Option<String>, host: &ImagePlatform, sysroot: &Path) -> Self {
+    pub fn new(
+        channel: &str,
+        date: &Option<String>,
+        host: &ImagePlatform,
+        sysroot: &Path,
+        is_custom: bool,
+    ) -> Self {
         let mut this = Self {
             channel: channel.to_owned(),
             date: date.clone(),
             host: host.clone(),
-            is_custom: false,
+            is_custom,
             full: if let Some(date) = date {
                 format!("{}-{}-{}", channel, date, host.target)
             } else {
@@ -105,14 +111,16 @@ impl QualifiedToolchain {
             },
             sysroot: sysroot.to_owned(),
         };
-        this.sysroot.set_file_name(&this.full);
+        if !is_custom {
+            this.sysroot.set_file_name(&this.full);
+        }
         this
     }
 
     /// Replace the host, does nothing if ran on a custom toolchain
     pub fn replace_host(&mut self, host: &ImagePlatform) -> &mut Self {
         if !self.is_custom {
-            *self = Self::new(&self.channel, &self.date, host, &self.sysroot);
+            *self = Self::new(&self.channel, &self.date, host, &self.sysroot, false);
             self.sysroot.set_file_name(&self.full);
         }
         self
@@ -168,8 +176,8 @@ impl QualifiedToolchain {
                 &build_date,
                 &ImagePlatform::from_target(host.into())?,
                 sysroot,
+                true,
             );
-            toolchain.is_custom = true;
             toolchain.full = name.to_owned();
             return Ok(toolchain);
         }
@@ -217,6 +225,7 @@ impl QualifiedToolchain {
             &date,
             &host,
             &self.sysroot,
+            false,
         ))
     }
 
@@ -256,7 +265,11 @@ impl QualifiedToolchain {
             Ok(_) | Err(_) if config.custom_toolchain() => {
                 QualifiedToolchain::custom(toolchain, &sysroot, config, msg_info)
             }
-            Ok(_) => eyre::bail!("toolchain is not fully qualified"),
+            Ok(_) => return Err(eyre::eyre!("toolchain is not fully qualified")
+                .with_note(|| "cross expects the toolchain to be a rustup installed toolchain")
+                .with_suggestion(|| {
+                    "if you're using a custom toolchain try setting `CROSS_CUSTOM_TOOLCHAIN=1` or install rust via rustup"
+            })),
             Err(e) => Err(e),
         }
     }


Test patch:
diff --git a/src/tests.rs b/src/tests.rs
index 02afac1..7a22907 100644
--- a/src/tests.rs
+++ b/src/tests.rs
@@ -86,6 +86,7 @@ release: {version}
                     &None,
                     &ImagePlatform::from_const_target(TargetTriple::X86_64UnknownLinuxGnu),
                     Path::new("/toolchains/xxxx-x86_64-unknown-linux-gnu"),
+                    false,
                 ),
                 &target_meta.0,
                 &target_meta.1,