Repository: davidlattimore/wild (rust)

Issue:
Add test configuration file
Was just talking with @mati865 and learned that Wild has been packaged for Arch Linux, but some tests needed to be [skipped](https://gitlab.archlinux.org/archlinux/packaging/packages/wild/-/blob/main/PKGBUILD?ref_type=heads#L40) due to using nightly rustc.

We discussed having a configuration file for the tests. I propose `test-config.toml` in the repository root. The config file wouldn't be checked in and would be git-ignored, so people could supply their own configuration file as they see fit. We would check in a `test-config.toml.example` and a `test-config-ci.toml`.

The two main things that currently would be good to control via such a file are:

* Whether to use nightly rustc. The cranelift backend test would need to be skipped if we weren't using nightly.
* Whether to cross compile tests for aarch64 and run with qemu. This is currently disabled by default and only enabled via setting the environment variable `WILD_TEST_CROSS=aarch64`. I think we perhaps should keep the environment variable. I guess environment variable takes precedence, but I'm not sure.

The goal here is to make it easier for people to just `cargo test` wild without needing a lot of esoteric things installed, but still allow core developers and CI to run those more unusual tests.

Pull request:
add test configuration support
close #626 

This patch adds two options for configuring test behavior:

- `rustc_channel`: Specifies which Rust compiler channel to use when running tests that build Rust code.
- `use_qemu`: Determines whether to run tests for architectures different from the host. This setting is overridden by the `$WILD_TEST_CROSS` environment variable.

These options are specified in TOML format and take effect by setting the path to the configuration file via an environment variable.

Additionally, an option was added to explicitly indicate that a test requires the nightly Rust compiler (`RequiresNightlyRustc`).

Patch:
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index da28969..3f772a7 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -48,6 +48,8 @@ jobs:
       image: ${{ matrix.container }}
 
     steps:
+      - run: echo "WILD_TEST_CONFIG=test-config-ci.toml" >> $GITHUB_ENV
+        if: ${{ env.CI == 'true' }}
       - run: echo "WILD_TEST_CROSS=aarch64" >> $GITHUB_ENV
         if: ${{ matrix.test-qemu }}
       - run: apt-get update && apt-get -y install gcc g++ clang lld curl bubblewrap binutils-aarch64-linux-gnu
diff --git a/.gitignore b/.gitignore
index 6ffe535..1a9dadc 100644
--- a/.gitignore
+++ b/.gitignore
@@ -12,3 +12,4 @@ heaptrack*.zst
 .idea
 .cargo/config.toml
 dhat-heap.json
+test-config.toml
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
index 81b6e81..6a70dd8 100644
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -59,6 +59,22 @@ disable cross compilation.
 
 Cross compilation is set up in docker/debian.Dockerfile.
 
+## Configuration file for tests
+
+Currently, the behavior for the following test options can be configured using the TOML format:
+
+- `rustc_channel`: Specifies which Rust compiler channel to use when running tests that build Rust code. The default value is "default", which means no explicit toolchain is specified.
+
+- `use_qemu`: Determines whether to run tests for architectures different from the host. This setting is overridden by the `$WILD_TEST_CROSS` environment variable. The default value is `false`.
+
+A sample configuration file is provided as `test-config.toml.sample`.
+By default, Wild uses `test-config.toml` as the configuration file.
+If you have written your configuration in a different file, specify its location using the `WILD_TEST_CONFIG` environment variable as follows:
+
+```sh
+WILD_TEST_CONFIG=path_to_config cargo test
+```
+
 ## GitHub workflow
 
 TL;DR: We're pretty relaxed. Feel free to force push or not. Squash, rebase, merge, whatever you
diff --git a/Cargo.lock b/Cargo.lock
index eb937c5..65f76ef 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -1173,6 +1173,15 @@ dependencies = [
  "serde",
 ]
 
+[[package]]
+name = "serde_spanned"
+version = "0.6.8"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "87607cb1398ed59d48732e575a4c28a7a8ebf2454b964fe3f224f2afc07909e1"
+dependencies = [
+ "serde",
+]
+
 [[package]]
 name = "sharded-offset-map"
 version = "0.2.0"
@@ -1326,11 +1335,26 @@ dependencies = [
  "once_cell",
 ]
 
+[[package]]
+name = "toml"
+version = "0.8.20"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "cd87a5cdd6ffab733b2f74bc4fd7ee5fff6634124999ac278c35fc78c6120148"
+dependencies = [
+ "serde",
+ "serde_spanned",
+ "toml_datetime",
+ "toml_edit",
+]
+
 [[package]]
 name = "toml_datetime"
 version = "0.6.8"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "0dd7358ecb8fc2f8d014bf86f6f638ce72ba252a2c3a2572f2a795f1d23efb41"
+dependencies = [
+ "serde",
+]
 
 [[package]]
 name = "toml_edit"
@@ -1339,6 +1363,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "17b4795ff5edd201c7cd6dca065ae59972ce77d1b80fa0a84d94950ece7d1474"
 dependencies = [
  "indexmap",
+ "serde",
+ "serde_spanned",
  "toml_datetime",
  "winnow",
 ]
@@ -1488,8 +1514,10 @@ dependencies = [
  "object",
  "os_info",
  "rstest",
+ "serde",
  "strum",
  "strum_macros",
+ "toml",
  "wait-timeout",
  "which",
 ]
diff --git a/wild/Cargo.toml b/wild/Cargo.toml
index b5f65f8..6a8a06b 100644
--- a/wild/Cargo.toml
+++ b/wild/Cargo.toml
@@ -39,6 +39,8 @@ rstest = "0.25.0"
 fd-lock = "4.0.4"
 strum = { version = "0.27.1", features = ["derive"] }
 strum_macros = "0.27.1"
+toml = "0.8.20"
+serde = { version = "1.0.219", features = ["derive"] }
 
 [features]
 default = ["fork"]


Test patch:
diff --git a/test-config-ci.toml b/test-config-ci.toml
new file mode 100644
index 0000000..8dab4e6
--- /dev/null
+++ b/test-config-ci.toml
@@ -0,0 +1,2 @@
+rustc_channel = "nightly"
+use_qemu = false
diff --git a/test-config.toml.sample b/test-config.toml.sample
new file mode 100644
index 0000000..f18fd1e
--- /dev/null
+++ b/test-config.toml.sample
@@ -0,0 +1,2 @@
+rustc_channel = FILLHERE # `"stable"`, `"beta"`, `"nightly"` or "default"
+use_qemu = FILLHERE # `true` or `false`
diff --git a/wild/tests/integration_tests.rs b/wild/tests/integration_tests.rs
index 5c2c36f..3c1743f 100644
--- a/wild/tests/integration_tests.rs
+++ b/wild/tests/integration_tests.rs
@@ -83,6 +83,9 @@
 //! RequiresGlibc:{bool} Defaults to false. Set to true to disable this test if we're running on a
 //! system without glibc.
 //!
+//! RequiresNightlyRustc:{bool} Defaults to false. Set to true to disable this test if we detect that the
+//! version of rustc available to us is not nightly.
+//!
 //! RequiresClangWithTlsDesc:{bool} Defaults to false. Set to true to disable this test if we detect
 //! that the version of clang available to us doesn't support TLSDESC.
 //!
@@ -414,7 +417,24 @@ struct Config {
     support_architectures: Vec<Architecture>,
     requires_glibc: bool,
     requires_clang_with_tlsdesc: bool,
+    requires_nightly_rustc: bool,
     version_script: Option<PathBuf>,
+    rustc_channel: Option<RustcChannel>,
+}
+
+#[derive(serde::Deserialize)]
+struct TestConfig {
+    // These configs are used by the config file specified in `$WILD_TEST_CONFIG`
+    rustc_channel: RustcChannel,
+    use_qemu: bool,
+}
+
+#[derive(Clone, Copy, PartialEq, Eq, serde::Deserialize, Debug)]
+#[serde(rename_all = "snake_case")]
+enum RustcChannel {
+    Stable,
+    Beta,
+    Nightly,
 }
 
 #[derive(Clone, Copy, PartialEq, Eq)]
@@ -429,12 +449,13 @@ impl Default for DirectConfig {
 }
 
 impl Config {
-    fn should_skip(&self, arch: Architecture) -> bool {
+    fn should_skip(&self, arch: Architecture, test_config: &TestConfig) -> bool {
         !self.support_architectures.contains(&arch)
             || self.requires_glibc && !cfg!(target_env = "gnu")
             || (self.requires_clang_with_tlsdesc && !host_supports_clang_with_tls_desc())
             || (arch != get_host_architecture()
                 && (self.compiler == "clang" || !self.cross_enabled))
+            || (test_config.rustc_channel != RustcChannel::Nightly && self.requires_nightly_rustc)
     }
 
     fn is_linker_enabled(&self, linker: &Linker) -> bool {
@@ -579,7 +600,9 @@ impl Default for Config {
             support_architectures: ALL_ARCHITECTURES.to_owned(),
             requires_glibc: false,
             requires_clang_with_tlsdesc: false,
+            requires_nightly_rustc: false,
             version_script: None,
+            rustc_channel: None,
         }
     }
 }
@@ -735,6 +758,9 @@ fn parse_configs(src_filename: &Path) -> Result<Vec<Config>> {
                 "RequiresClangWithTlsDesc" => {
                     config.requires_clang_with_tlsdesc = arg.to_lowercase().parse()?;
                 }
+                "RequiresNightlyRustc" => {
+                    config.requires_nightly_rustc = arg.to_lowercase().parse()?;
+                }
                 "VersionScript" => {
                     config.version_script = Some(src_path(&arg.trim().to_lowercase()))
                 }
@@ -742,6 +768,7 @@ fn parse_configs(src_filename: &Path) -> Result<Vec<Config>> {
             }
         }
     }
+
     let mut configs = config_by_name
         .into_values()
         .filter(|c| !c.is_abstract)
@@ -1072,10 +1099,16 @@ fn build_obj(
         }
         CompilerKind::Rust => {
             let wild = wild_path().to_str().context("Need UTF-8 path")?.to_owned();
+            let rustc_channel = match config.rustc_channel {
+                Some(RustcChannel::Stable) => Some("+stable"),
+                Some(RustcChannel::Beta) => Some("+beta"),
+                Some(RustcChannel::Nightly) => Some("+nightly"),
+                None => None,
+            };
 
             command
                 .env("WILD_SAVE_SKIP_LINKING", "1")
-                .arg("+nightly")
+                .args(rustc_channel)
                 .args(["-C", "linker=clang"])
                 .args(["-C", &format!("link-arg=--ld-path={wild}")]);
 
@@ -2187,21 +2220,71 @@ fn integration_test(
 
     let host_arch = get_host_architecture();
 
-    // We only currently support cross compilation from x86_64 to aarch64, so we don't need to track
-    // which targets are enabled, since there's only one.
-    let cross_enabled = std::env::var("WILD_TEST_CROSS").is_ok_and(|v| v == "aarch64");
+    let test_config = read_test_config()?;
 
     for &arch in ALL_ARCHITECTURES {
-        if arch != host_arch && !cross_enabled {
+        if arch != host_arch && !test_config.use_qemu {
             continue;
         }
 
-        let config_it = configs.iter().filter(|config| !config.should_skip(arch));
+        let config_it = configs
+            .iter()
+            .filter(|config| !config.should_skip(arch, &test_config));
 
         for config in config_it {
-            run_with_config(&program_inputs, config, arch, &linkers)?
+            let mut config = config.clone();
+            config.rustc_channel = Some(test_config.rustc_channel);
+            run_with_config(&program_inputs, &config, arch, &linkers)?
         }
     }
 
     Ok(())
 }
+
+fn read_test_config() -> Result<TestConfig> {
+    // We only currently support cross compilation from x86_64 to aarch64, so we don't need to track
+    // which targets are enabled, since there's only one.
+    let mut use_qemu = std::env::var("WILD_TEST_CROSS").is_ok_and(|v| v == "aarch64");
+    let mut rustc_channel = RustcChannel::Nightly;
+
+    let config_default_path = std::path::Path::new(env!("CARGO_MANIFEST_DIR"))
+        .parent()
+        .unwrap()
+        .join("test-config.toml");
+    let config_path = std::env::var("WILD_TEST_CONFIG")
+        .map(|config_path| {
+            std::path::Path::new(env!("CARGO_MANIFEST_DIR"))
+                .parent()
+                .unwrap()
+                .join(config_path)
+        })
+        .unwrap_or_else(|_| config_default_path.clone());
+
+    if config_path.exists() {
+        let config_content = std::fs::read_to_string(&config_path).with_context(|| {
+            format!(
+                "Failed to read WILD_TEST_CONFIG file at `{}`",
+                config_path.display()
+            )
+        })?;
+        let data: TestConfig = match toml::from_str(&config_content) {
+            Ok(d) => d,
+            Err(_) => {
+                bail!("Unable to load config from {:?}", config_path);
+            }
+        };
+
+        rustc_channel = data.rustc_channel;
+        use_qemu |= data.use_qemu;
+    } else if config_path != config_default_path {
+        bail!(
+            "WILD_TEST_CONFIG file not found at `{}`",
+            config_path.display()
+        );
+    }
+
+    Ok(TestConfig {
+        rustc_channel,
+        use_qemu,
+    })
+}
diff --git a/wild/tests/sources/rust-integration.rs b/wild/tests/sources/rust-integration.rs
index 04e0e44..5e20638 100644
--- a/wild/tests/sources/rust-integration.rs
+++ b/wild/tests/sources/rust-integration.rs
@@ -14,6 +14,7 @@
 
 //#Config:cranelift-static:default
 //#CompArgs:-Zcodegen-backend=cranelift --target x86_64-unknown-linux-musl -C relocation-model=static -C target-feature=+crt-static -C debuginfo=2 --cfg cranelift
+//#RequiresNightlyRustc: true
 //#Arch: x86_64
 
 //#Config:cranelift-static-aarch64:default
diff --git a/wild/tests/sources/rust-tls.rs b/wild/tests/sources/rust-tls.rs
index 6453241..fe9b0ee 100644
--- a/wild/tests/sources/rust-tls.rs
+++ b/wild/tests/sources/rust-tls.rs
@@ -1,4 +1,5 @@
 //#AbstractConfig:default
+//#RequiresNightlyRustc: true
 
 //#Config:global-dynamic:default
 //#CompArgs:-Ztls-model=global-dynamic