Repository: probe-rs/probe-rs
Pull request #2339 (2024-04-03 19:20:54)

Try out some smoke tester improvements

noppej: @Tiwalun I know this is a draft, but it would help me if perhaps you can explain the role you see for the smoke-tester going forward.  In my experience, 'smoke testing' is generally a "make sure the basics work, before running more comprehensive tests". Is that what you intend here?  What do you see as the vision for the ultimate testing infrastructure for probe-rs? e.g. "build", "smoke-test", ... then what?
Tiwalun: > @Tiwalun I know this is a draft, but it would help me if perhaps you can explain the role you see for the smoke-tester going forward. In my experience, 'smoke testing' is generally a "make sure the basics work, before running more comprehensive tests". Is that what you intend here? What do you see as the vision for the ultimate testing infrastructure for probe-rs? e.g. "build", "smoke-test", ... then what?

The name is probably a bit misleading, I didn't put too much thought into it. The plan is to expand the scope of test for the library, and try and cover more use cases. So it should be more of a general test framework, not some kind of limited "smoke test" setup.

Patch:
diff --git a/Cargo.lock b/Cargo.lock
index 92c14e4f..c7b74d0f 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -286,6 +286,15 @@ dependencies = [
  "rustc-demangle",
 ]
 
+[[package]]
+name = "backtrace-ext"
+version = "0.2.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "537beee3be4a18fb023b570f80e3ae28003db9167a751266b259926e25539d50"
+dependencies = [
+ "backtrace",
+]
+
 [[package]]
 name = "base64"
 version = "0.21.7"
@@ -2082,6 +2091,12 @@ dependencies = [
  "windows-sys 0.52.0",
 ]
 
+[[package]]
+name = "is_ci"
+version = "1.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7655c9839580ee829dfacba1d1278c2b7883e50a277ff7541299489d6bdfdc45"
+
 [[package]]
 name = "itertools"
 version = "0.4.19"
@@ -2330,8 +2345,16 @@ version = "7.2.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "4edc8853320c2a0dab800fbda86253c8938f6ea88510dc92c5f1ed20e794afc1"
 dependencies = [
+ "backtrace",
+ "backtrace-ext",
  "cfg-if",
  "miette-derive",
+ "owo-colors",
+ "supports-color",
+ "supports-hyperlinks",
+ "supports-unicode",
+ "terminal_size",
+ "textwrap",
  "thiserror",
  "unicode-width",
 ]
@@ -2548,6 +2571,12 @@ version = "0.1.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "b15813163c1d831bf4a13c3610c05c0d03b39feb07f7e09fa234dac9b15aaf39"
 
+[[package]]
+name = "owo-colors"
+version = "4.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "caff54706df99d2a78a5a4e3455ff45448d81ef1bb63c22cd14052ca0e993a3f"
+
 [[package]]
 name = "parking"
 version = "2.2.0"
@@ -3776,14 +3805,15 @@ checksum = "b7c388c1b5e93756d0c740965c41e8822f866621d41acbdf6336a6a168f8840c"
 name = "smoke_tester"
 version = "0.23.0"
 dependencies = [
- "anyhow",
  "clap",
  "colored",
  "linkme",
  "log",
+ "miette",
  "pretty_env_logger",
  "probe-rs",
  "serde",
+ "thiserror",
  "toml",
 ]
 
@@ -3865,6 +3895,27 @@ version = "2.5.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "81cdd64d312baedb58e21336b31bc043b77e01cc99033ce76ef539f78e965ebc"
 
+[[package]]
+name = "supports-color"
+version = "3.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9829b314621dfc575df4e409e79f9d6a66a3bd707ab73f23cb4aa3a854ac854f"
+dependencies = [
+ "is_ci",
+]
+
+[[package]]
+name = "supports-hyperlinks"
+version = "3.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2c0a1e5168041f5f3ff68ff7d95dcb9c8749df29f6e7e89ada40dd4c9de404ee"
+
+[[package]]
+name = "supports-unicode"
+version = "3.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b7401a30af6cb5818bb64852270bb722533397edcfc7344954a38f420819ece2"
+
 [[package]]
 name = "svd-parser"
 version = "0.14.5"
@@ -4008,6 +4059,16 @@ dependencies = [
  "winapi-util",
 ]
 
+[[package]]
+name = "terminal_size"
+version = "0.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "21bebf2b7c9e0a515f6e0f8c51dc0f8e4696391e6f1ff30379559f8365fb0df7"
+dependencies = [
+ "rustix",
+ "windows-sys 0.48.0",
+]
+
 [[package]]
 name = "termtree"
 version = "0.4.1"


Test patch:
diff --git a/.github/workflows/smoketest.yml b/.github/workflows/smoketest.yml
index 64d0f9d5..6190acff 100644
--- a/.github/workflows/smoketest.yml
+++ b/.github/workflows/smoketest.yml
@@ -52,4 +52,4 @@ jobs:
       - name: Run smoke-tester
         run:  |
           chmod +x smoke_tester
-          ./smoke_tester --dut-definitions $SMOKE_TESTER_CONFIG --markdown-summary $GITHUB_STEP_SUMMARY
+          ./smoke_tester test --dut-definitions $SMOKE_TESTER_CONFIG --markdown-summary $GITHUB_STEP_SUMMARY
diff --git a/smoke-tester/Cargo.toml b/smoke-tester/Cargo.toml
index 8adc77a1..25a81d35 100644
--- a/smoke-tester/Cargo.toml
+++ b/smoke-tester/Cargo.toml
@@ -9,12 +9,14 @@ publish = false
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
+
+serde = { version = "1", features = ["derive"] }
 probe-rs = { workspace = true }
 toml = "0.8.12"
-anyhow = "1.0.82"
-serde = { version = "1", features = ["derive"] }
 pretty_env_logger = "0.5.0"
 log = "0.4.21"
-clap = "4.5"
+clap = { version = "4.5", features = ["derive"] }
 colored = "2.1.0"
 linkme = "0.3.25"
+miette = { version = "7.2.0", features = ["fancy"] }
+thiserror.workspace = true
diff --git a/smoke-tester/src/dut_definition.rs b/smoke-tester/src/dut_definition.rs
index 96d1ae45..50758ec1 100644
--- a/smoke-tester/src/dut_definition.rs
+++ b/smoke-tester/src/dut_definition.rs
@@ -3,9 +3,11 @@
 //! This module handles the definition of the different devices under test (DUTs),
 //! which are used by the tester.
 
-use anyhow::{bail, ensure, Context, Result};
+use miette::IntoDiagnostic;
+use miette::Result;
+use miette::WrapErr;
 use probe_rs::{
-    config::{get_target_by_name, search_chips},
+    config::get_target_by_name,
     probe::{list::Lister, DebugProbeSelector, Probe},
     Target,
 };
@@ -31,21 +33,21 @@ struct RawDutDefinition {
 impl RawDutDefinition {
     /// Try to parse a DUT definition from a file.
     fn from_file(file: &Path) -> Result<Self> {
-        let file_content = std::fs::read_to_string(file)?;
+        let file_content = std::fs::read_to_string(file).into_diagnostic()?;
 
-        let definition: RawDutDefinition = toml::from_str(&file_content)?;
+        let definition: RawDutDefinition = toml::from_str(&file_content).into_diagnostic()?;
 
         Ok(definition)
     }
 }
 
-#[derive(Clone)]
+#[derive(Debug, Clone)]
 pub enum DefinitionSource {
     File(PathBuf),
     Cli,
 }
 
-#[derive(Clone)]
+#[derive(Debug, Clone)]
 pub struct DutDefinition {
     pub chip: Target,
 
@@ -69,10 +71,10 @@ pub struct DutDefinition {
 }
 
 impl DutDefinition {
-    pub fn new(chip: &str, probe: &str) -> Result<Self> {
+    pub fn new(chip: &str, probe: &str) -> miette::Result<Self> {
         let target = lookup_unique_target(chip)?;
 
-        let selector: DebugProbeSelector = probe.parse()?;
+        let selector: DebugProbeSelector = probe.parse().into_diagnostic()?;
 
         Ok(DutDefinition {
             chip: target,
@@ -83,7 +85,7 @@ impl DutDefinition {
         })
     }
 
-    pub fn autodetect_probe(chip: &str) -> Result<Self> {
+    pub fn autodetect_probe(chip: &str) -> miette::Result<Self> {
         let target = lookup_unique_target(chip)?;
 
         Ok(DutDefinition {
@@ -106,7 +108,7 @@ impl DutDefinition {
     pub fn collect(directory: impl AsRef<Path>) -> Result<Vec<DutDefinition>> {
         let directory = directory.as_ref();
 
-        ensure!(
+        miette::ensure!(
             directory.is_dir(),
             "Unable to collect target definitions from path '{}'. Path is not a directory.",
             directory.display()
@@ -114,8 +116,8 @@ impl DutDefinition {
 
         let mut definitions = Vec::new();
 
-        for file in directory.read_dir()? {
-            let file_path = file?.path();
+        for file in directory.read_dir().into_diagnostic()? {
+            let file_path = file.into_diagnostic()?.path();
 
             // Ignore files without .toml ending
             if file_path.extension() != Some(OsStr::new("toml")) {
@@ -126,8 +128,9 @@ impl DutDefinition {
                 continue;
             }
 
-            let definition = DutDefinition::from_file(&file_path)
-                .with_context(|| format!("Failed to parse definition '{}'", file_path.display()))?;
+            let definition = DutDefinition::from_file(&file_path).wrap_err_with(|| {
+                format!("Failed to parse definition '{}'", file_path.display())
+            })?;
 
             definitions.push(definition);
         }
@@ -149,21 +152,22 @@ impl DutDefinition {
             Some(selector) => {
                 let probe = lister
                     .open(selector)
-                    .with_context(|| format!("Failed to open probe with selector {selector}"))?;
+                    .into_diagnostic()
+                    .wrap_err_with(|| format!("Failed to open probe with selector {selector}"))?;
 
                 Ok(probe)
             }
             None => {
                 let probes = lister.list_all();
 
-                ensure!(!probes.is_empty(), "No probes detected!");
+                miette::ensure!(!probes.is_empty(), "No probes detected!");
 
-                ensure!(
+                miette::ensure!(
                     probes.len() < 2,
                     "Multiple probes detected. Specify which probe to use using the '--probe' argument."
                 );
 
-                let probe = probes[0].open()?;
+                let probe = probes[0].open().into_diagnostic()?;
 
                 Ok(probe)
             }
@@ -171,7 +175,7 @@ impl DutDefinition {
     }
 
     fn from_raw_definition(raw_definition: RawDutDefinition, source_file: &Path) -> Result<Self> {
-        let probe_selector = Some(raw_definition.probe_selector.try_into()?);
+        let probe_selector = Some(raw_definition.probe_selector.try_into().into_diagnostic()?);
 
         let target = lookup_unique_target(&raw_definition.chip)?;
 
@@ -190,38 +194,33 @@ impl DutDefinition {
 }
 
 fn lookup_unique_target(chip: &str) -> Result<Target> {
-    let targets = search_chips(chip)?
-        .into_iter()
-        .map(|chip| chip.to_ascii_lowercase())
-        .collect::<Vec<_>>();
-
-    ensure!(
-        !targets.is_empty(),
-        "Unable to find any chip matching {}",
-        &chip
-    );
-
-    if targets.len() > 1 {
-        let target_string = chip.to_ascii_lowercase();
-        if targets.contains(&target_string) {
-            // Multiple chips returned, but one was an exact match so we're using it
-            let target = get_target_by_name(target_string)?;
-            return Ok(target);
-        } else {
-            eprintln!(
-                "For tests, chip definition must be exact. Chip name {} matches multiple chips:",
-                &chip
-            );
-
-            for target in &targets {
-                eprintln!("\t{target}");
-            }
+    let target = get_target_by_name(chip).into_diagnostic()?;
 
-            bail!("Chip definition does not match exactly.");
-        }
+    if !target.name.eq_ignore_ascii_case(chip) {
+        miette::bail!("Chip definition does not match exactly, the chip is specified as {}, but the entry in the registry is {}", chip, target.name);
     }
 
-    let target = get_target_by_name(&targets[0])?;
-
     Ok(target)
 }
+
+#[test]
+fn find_unique_target() {
+    let target = lookup_unique_target("nRF52840_xxAA").unwrap();
+
+    assert_eq!(target.name, "nRF52840_xxAA");
+}
+
+#[test]
+fn find_unique_target_failure() {
+    // this should fail because the full name is nRF52840_xxAA
+    lookup_unique_target("nRF52840_xx").unwrap_err();
+}
+
+#[test]
+fn find_unique_target_with_non_unique_prefix() {
+    // There is also a chip named "esp32c6_lp" in the registry, this ensures
+    // that looking up just "esp32c6" still works.
+    let target = lookup_unique_target("esp32c6").unwrap();
+
+    assert_eq!(target.name, "esp32c6");
+}
diff --git a/smoke-tester/src/macros.rs b/smoke-tester/src/macros.rs
index 2209f00d..df19acf4 100644
--- a/smoke-tester/src/macros.rs
+++ b/smoke-tester/src/macros.rs
@@ -37,7 +37,7 @@ macro_rules! println_dut_status {
 #[macro_export]
 macro_rules! println_test_status {
     ($tracker:expr, $color:ident,$($arg:tt)*) => ({
-        let prefix = format!("[{}/{}]({}) - Test [{}/{}]", $tracker.current_dut(), $tracker.num_duts(), $tracker.current_dut_name(), $tracker.current_test(), $tracker.num_tests()).$color();
+        let prefix = format!("[{}/{}]({}) - Test [{}]", $tracker.current_dut(), $tracker.num_duts(), $tracker.current_dut_name(), $tracker.current_test()).$color();
         print!("{} - ", prefix);
         println!($($arg)*);
     })
diff --git a/smoke-tester/src/main.rs b/smoke-tester/src/main.rs
index 3db1dcc1..048d1243 100644
--- a/smoke-tester/src/main.rs
+++ b/smoke-tester/src/main.rs
@@ -1,85 +1,121 @@
 use std::{
-    error::Error,
     io::Write,
-    marker::PhantomData,
-    path::Path,
+    path::{Path, PathBuf},
     process::ExitCode,
     time::{Duration, Instant},
 };
 
-use crate::{
-    dut_definition::{DefinitionSource, DutDefinition},
-    tests::test_flashing,
-};
-use anyhow::{Context, Result};
-use colored::Colorize;
+use crate::dut_definition::{DefinitionSource, DutDefinition};
 
-use clap::{Arg, Command};
+use clap::{Parser, Subcommand};
+use colored::Colorize;
 use linkme::distributed_slice;
+use miette::{Context, IntoDiagnostic, Result};
 use probe_rs::Permissions;
 
 mod dut_definition;
 mod macros;
 mod tests;
 
+#[derive(Debug, miette::Diagnostic, thiserror::Error)]
+pub enum TestFailure {
+    #[error("Test returned an error")]
+    Error(#[from] Box<dyn std::error::Error + Send + Sync + 'static>),
+
+    #[error("The test was skipped: {0}")]
+    Skipped(String),
+
+    #[error("Test is not implemented for target {0:?}: {1}")]
+    UnimplementedForTarget(Box<probe_rs::Target>, String),
+    #[error("A resource necessary to execute the test is not available: {0}")]
+    MissingResource(String),
+
+    /// A fatal error means that all future tests will be cancelled as well.
+    #[error("A fatal error occured")]
+    Fatal(#[source] Box<dyn std::error::Error + Send + Sync + 'static>),
+}
+
+impl From<miette::ErrReport> for TestFailure {
+    fn from(report: miette::ErrReport) -> Self {
+        TestFailure::Error(report.into())
+    }
+}
+
+impl From<probe_rs::Error> for TestFailure {
+    fn from(error: probe_rs::Error) -> Self {
+        TestFailure::Error(error.into())
+    }
+}
+
+pub type TestResult = Result<(), TestFailure>;
+
+#[derive(Debug)]
+struct SingleTestReport {
+    result: TestResult,
+    _duration: Duration,
+}
+
+impl SingleTestReport {
+    fn has_fatal_error(&self) -> bool {
+        matches!(self.result, Err(TestFailure::Fatal(_)))
+    }
+}
+
+#[derive(Debug, Subcommand)]
+enum Command {
+    Test {
+        #[arg(long, value_name = "FILE")]
+        markdown_summary: Option<PathBuf>,
+    },
+}
+
+#[derive(Debug, Parser)]
+struct Opt {
+    #[command(subcommand)]
+    command: Command,
+
+    #[arg(long, global = true, value_name = "DIRECTORY", conflicts_with_all = ["chip", "probe", "single_dut"])]
+    dut_definitions: Option<PathBuf>,
+
+    #[arg(long, global = true, value_name = "CHIP", conflicts_with_all = ["dut_definitions", "single_dut"])]
+    chip: Option<String>,
+
+    #[arg(long, global = true, value_name = "PROBE")]
+    probe: Option<String>,
+
+    #[arg(long, global = true, value_name = "FILE", conflicts_with_all = ["chip", "dut_definitions"])]
+    single_dut: Option<PathBuf>,
+}
+
 fn main() -> Result<ExitCode> {
     pretty_env_logger::init();
 
-    let app = Command::new("smoke tester")
-        .arg(
-            Arg::new("dut_definitions")
-                .long("dut-definitions")
-                .value_name("DIRECTORY")
-                .conflicts_with_all(["chip", "probe", "single_dut"])
-                .required(true),
-        )
-        .arg(
-            Arg::new("chip")
-                .long("chip")
-                .value_name("CHIP")
-                .conflicts_with_all(["dut_definitions", "single_dut"])
-                .required(true),
-        )
-        .arg(
-            Arg::new("probe")
-                .long("probe")
-                .value_name("PROBE")
-                .required(false),
-        )
-        .arg(
-            Arg::new("single_dut")
-                .long("single-dut")
-                .value_name("FILE")
-                .required(true)
-                .conflicts_with_all(["chip", "dut_definitions"]),
-        )
-        .arg(
-            Arg::new("markdown_summary")
-                .long("markdown-summary")
-                .value_name("FILE")
-                .required(false),
-        );
-
-    let matches = app.get_matches();
-
-    let definitions = if let Some(dut_definitions) = matches.get_one::<String>("dut_definitions") {
+    let opt = Opt::parse();
+
+    let definitions = if let Some(dut_definitions) = opt.dut_definitions.as_deref() {
         let definitions = DutDefinition::collect(dut_definitions)?;
         println!("Found {} target definitions.", definitions.len());
         definitions
-    } else if let Some(single_dut) = matches.get_one::<String>("single_dut") {
+    } else if let Some(single_dut) = opt.single_dut.as_deref() {
         vec![DutDefinition::from_file(Path::new(single_dut))?]
     } else {
         // Chip needs to be specified
-        let chip = matches.get_one::<String>("chip").unwrap(); // If dut-definitions is not present, chip must be present
+        let chip = opt.chip.as_deref().unwrap(); // If dut-definitions is not present, chip must be present
 
-        if let Some(probe) = matches.get_one::<String>("probe") {
+        if let Some(probe) = &opt.probe {
             vec![DutDefinition::new(chip, probe)?]
         } else {
             vec![DutDefinition::autodetect_probe(chip)?]
         }
     };
 
-    let mut test_tracker = TestTracker::new(definitions, 0);
+    match opt.command {
+        Command::Test { markdown_summary } => run_test(&definitions, markdown_summary),
+    }
+}
+
+fn run_test(definitions: &[DutDefinition], markdown_summary: Option<PathBuf>) -> Result<ExitCode> {
+    let mut test_tracker = TestTracker::new(definitions);
 
     let result = test_tracker.run(|tracker, definition| {
         let probe = definition.open_probe()?;
@@ -92,7 +128,8 @@ fn main() -> Result<ExitCode> {
 
         let mut session = probe
             .attach(definition.chip.clone(), permissions)
-            .context("Failed to attach to chip")?;
+            .into_diagnostic()
+            .wrap_err("Failed to attach to chip")?;
         let cores = session.list_cores();
 
         // TODO: Handle different cores. Handling multiple cores is not supported properly yet,
@@ -100,25 +137,35 @@ fn main() -> Result<ExitCode> {
         for (core_index, core_type) in cores.into_iter().take(1) {
             println_dut_status!(tracker, blue, "Core {}: {:?}", core_index, core_type);
 
-            let mut core = session.core(core_index)?;
+            let mut core = session.core(core_index).into_diagnostic()?;
 
             println_dut_status!(tracker, blue, "Halting core..");
 
-            core.reset_and_halt(Duration::from_millis(500))?;
+            core.reset_and_halt(Duration::from_millis(500))
+                .into_diagnostic()?;
 
             for test_fn in CORE_TESTS {
-                tracker.run_test(|tracker| test_fn(tracker, &mut core))?;
+                let result = tracker.run_test(|tracker| test_fn(tracker, &mut core));
+
+                if result.has_fatal_error() {
+                    return Err(miette::miette!("Test failed with fatal error"));
+                }
             }
 
             // Ensure core is not running anymore.
-            core.reset_and_halt(Duration::from_millis(200))?;
+            core.reset_and_halt(Duration::from_millis(200))
+                .into_diagnostic()
+                .wrap_err_with(|| {
+                    format!("Failed to reset core with index {core_index} after test")
+                })?;
         }
 
-        if let Some(flash_binary) = &definition.flash_test_binary {
-            tracker.run_test(|tracker| {
-                test_flashing(tracker, &mut session, flash_binary)?;
-                Ok(())
-            })?;
+        for test in SESSION_TESTS {
+            let result = tracker.run_test(|tracker| test(tracker, &mut session));
+
+            if result.has_fatal_error() {
+                return Err(miette::miette!("Test failed with fatal error"));
+            }
         }
 
         drop(session);
@@ -128,8 +175,9 @@ fn main() -> Result<ExitCode> {
         if definition.reset_connected {
             let probe = definition.open_probe()?;
 
-            let _session =
-                probe.attach_under_reset(definition.chip.clone(), Permissions::default())?;
+            let _session = probe
+                .attach_under_reset(definition.chip.clone(), Permissions::default())
+                .into_diagnostic()?;
         }
 
         Ok(())
@@ -139,17 +187,26 @@ fn main() -> Result<ExitCode> {
 
     let printer = ConsoleReportPrinter;
 
-    printer.print(&result, std::io::stdout())?;
-
-    if let Some(summary_file) = matches.get_one::<String>("markdown_summary") {
-        let mut file = std::fs::File::create(summary_file)?;
+    printer
+        .print(&result, std::io::stdout())
+        .into_diagnostic()?;
+
+    if let Some(summary_file) = &markdown_summary {
+        let mut file = std::fs::File::create(summary_file)
+            .into_diagnostic()
+            .wrap_err_with(|| {
+                format!(
+                    "Failed to create markdown summary file at location {}",
+                    summary_file.display()
+                )
+            })?;
 
-        writeln!(file, "## smoke-tester")?;
+        writeln!(file, "## smoke-tester").into_diagnostic()?;
 
         for dut in &result.dut_tests {
             let test_state = if dut.succesful { "Passed" } else { "Failed" };
 
-            writeln!(file, " - {}: {}", dut.name, test_state)?;
+            writeln!(file, " - {}: {}", dut.name, test_state).into_diagnostic()?;
         }
     }
 
@@ -236,22 +293,19 @@ impl ConsoleReportPrinter {
     }
 }
 
-pub struct TestTracker<'a> {
-    dut_definitions: Vec<DutDefinition>,
+#[derive(Debug)]
+pub struct TestTracker<'dut> {
+    dut_definitions: &'dut [DutDefinition],
     current_dut: usize,
-    num_tests: usize,
     current_test: usize,
-    _marker: PhantomData<&'a ()>,
 }
 
-impl<'a> TestTracker<'a> {
-    fn new(dut_definitions: Vec<DutDefinition>, num_tests: usize) -> Self {
+impl<'dut> TestTracker<'dut> {
+    fn new(dut_definitions: &'dut [DutDefinition]) -> Self {
         Self {
             dut_definitions,
             current_dut: 0,
-            num_tests,
             current_test: 0,
-            _marker: PhantomData,
         }
     }
 
@@ -276,26 +330,28 @@ impl<'a> TestTracker<'a> {
         self.current_test + 1
     }
 
-    fn num_tests(&self) -> usize {
-        self.num_tests
-    }
-
     fn advance_test(&mut self) {
         self.current_test += 1;
     }
 
+    pub fn current_target(&self) -> &probe_rs::Target {
+        &self.dut_definitions[self.current_dut].chip
+    }
+
+    pub fn current_dut_definition(&self) -> &DutDefinition {
+        &self.dut_definitions[self.current_dut]
+    }
+
     #[must_use]
     fn run(
         &mut self,
-        handle_dut: impl Fn(&mut TestTracker, &DutDefinition) -> Result<(), probe_rs::Error>
-            + Sync
-            + Send,
+        handle_dut: impl Fn(&mut TestTracker, &DutDefinition) -> miette::Result<()> + Sync + Send,
     ) -> TestReport {
         let mut report = TestReport::new();
 
         let mut tests_ok = true;
 
-        for definition in &self.dut_definitions.clone() {
+        for definition in self.dut_definitions {
             print_dut_status!(self, blue, "Starting Test",);
 
             if let DefinitionSource::File(path) = &definition.source {
@@ -354,8 +410,8 @@ impl<'a> TestTracker<'a> {
 
     fn run_test(
         &mut self,
-        test: impl FnOnce(&TestTracker) -> Result<(), probe_rs::Error>,
-    ) -> Result<(), probe_rs::Error> {
+        test: impl FnOnce(&TestTracker) -> Result<(), TestFailure>,
+    ) -> SingleTestReport {
         let start_time = Instant::now();
 
         let test_result = test(self);
@@ -372,16 +428,35 @@ impl<'a> TestTracker<'a> {
             Ok(()) => {
                 println_test_status!(self, green, "Test passed in {formatted_duration}.");
             }
+            Err(TestFailure::UnimplementedForTarget(target, message)) => {
+                println_test_status!(
+                    self,
+                    yellow,
+                    "Test not implemented for {}: {}",
+                    target.name,
+                    message
+                );
+            }
+            Err(TestFailure::MissingResource(message)) => {
+                println_test_status!(self, yellow, "Missing resource for test: {}", message);
+            }
             Err(_e) => {
                 println_test_status!(self, red, "Test failed in {formatted_duration}.");
             }
         };
         self.advance_test();
 
-        test_result
+        SingleTestReport {
+            result: test_result,
+            _duration: duration,
+        }
     }
 }
 
 /// A list of all tests which run on cores.
 #[distributed_slice]
-pub static CORE_TESTS: [fn(&TestTracker, &mut probe_rs::Core) -> Result<(), probe_rs::Error>];
+pub static CORE_TESTS: [fn(&TestTracker, &mut probe_rs::Core) -> TestResult];
+
+/// A list of all tests which run on `Session`.
+#[distributed_slice]
+pub static SESSION_TESTS: [fn(&TestTracker, &mut probe_rs::Session) -> TestResult];
diff --git a/smoke-tester/src/tests.rs b/smoke-tester/src/tests.rs
index 591709d9..ea7dd29e 100644
--- a/smoke-tester/src/tests.rs
+++ b/smoke-tester/src/tests.rs
@@ -1,20 +1,21 @@
-use std::{path::Path, time::Instant};
+use std::time::Instant;
 
 use colored::Colorize;
 use linkme::distributed_slice;
 use probe_rs::{
+    config::MemoryRegion,
     flashing::{download_file_with_options, DownloadOptions, FlashProgress, Format},
     Architecture, Core, MemoryInterface, Session,
 };
 
 pub mod stepping;
 
-use anyhow::{Context, Result};
+use miette::{IntoDiagnostic, Result, WrapErr};
 
-use crate::{println_test_status, TestTracker, CORE_TESTS};
+use crate::{println_test_status, TestFailure, TestResult, TestTracker, CORE_TESTS, SESSION_TESTS};
 
 #[distributed_slice(CORE_TESTS)]
-pub fn test_register_read(tracker: &TestTracker, core: &mut Core) -> Result<(), probe_rs::Error> {
+pub fn test_register_read(tracker: &TestTracker, core: &mut Core) -> TestResult {
     println_test_status!(tracker, blue, "Testing register read...");
 
     let register = core.registers();
@@ -22,6 +23,7 @@ pub fn test_register_read(tracker: &TestTracker, core: &mut Core) -> Result<(),
     for register in register.core_registers() {
         let _: u64 = core
             .read_core_reg(register)
+            .into_diagnostic()
             .with_context(|| format!("Failed to read register {}", register.name()))?;
     }
 
@@ -29,7 +31,7 @@ pub fn test_register_read(tracker: &TestTracker, core: &mut Core) -> Result<(),
 }
 
 #[distributed_slice(CORE_TESTS)]
-fn test_register_write(tracker: &TestTracker, core: &mut Core) -> Result<(), probe_rs::Error> {
+fn test_register_write(tracker: &TestTracker, core: &mut Core) -> TestResult {
     println_test_status!(tracker, blue, "Testing register write...");
 
     let register = core.registers();
@@ -54,9 +56,10 @@ fn test_register_write(tracker: &TestTracker, core: &mut Core) -> Result<(), pro
 
         // Write new value
 
-        core.write_core_reg(register, test_value)?;
+        core.write_core_reg(register, test_value)
+            .into_diagnostic()?;
 
-        let readback: u64 = core.read_core_reg(register)?;
+        let readback: u64 = core.read_core_reg(register).into_diagnostic()?;
 
         assert_eq!(
             test_value, readback,
@@ -70,114 +73,135 @@ fn test_register_write(tracker: &TestTracker, core: &mut Core) -> Result<(), pro
 }
 
 #[distributed_slice(CORE_TESTS)]
-fn test_memory_access(tracker: &TestTracker, core: &mut Core) -> Result<(), probe_rs::Error> {
-    let memory_regions = core.memory_regions().cloned().collect::<Vec<_>>();
+fn test_memory_access(tracker: &TestTracker, core: &mut Core) -> TestResult {
+    let memory_regions = core
+        .memory_regions()
+        .filter_map(|region| match region {
+            MemoryRegion::Ram(ram) => Some(ram),
+            _ => None,
+        })
+        .cloned()
+        .collect::<Vec<_>>();
 
     // Try to write all memory regions
-    for region in memory_regions {
-        match region {
-            probe_rs::config::MemoryRegion::Ram(ram) => {
-                let ram_start = ram.range.start;
-                let ram_size = ram.range.end - ram.range.start;
-
-                println_test_status!(tracker, blue, "Test - RAM Start 32");
-                // Write first word
-                core.write_word_32(ram_start, 0xababab)?;
-                let value = core.read_word_32(ram_start)?;
-                assert_eq!(value, 0xababab);
-
-                println_test_status!(tracker, blue, "Test - RAM End 32");
-                // Write last word
-                core.write_word_32(ram_start + ram_size - 4, 0xababac)?;
-                let value = core.read_word_32(ram_start + ram_size - 4)?;
-                assert_eq!(value, 0xababac);
-
-                println_test_status!(tracker, blue, "Test - RAM Start 8");
-                // Write first byte
-                core.write_word_8(ram_start, 0xac)?;
-                let value = core.read_word_8(ram_start)?;
-                assert_eq!(value, 0xac);
-
-                println_test_status!(tracker, blue, "Test - RAM 8 Unaligned");
-                let address = ram_start + 1;
-                let data = 0x23;
-                // Write last byte
-                core.write_word_8(address, data)
-                    .with_context(|| format!("Write_word_8 to address {address:08x}"))?;
-
-                let value = core
-                    .read_word_8(address)
-                    .with_context(|| format!("read_word_8 from address {address:08x}"))?;
-                assert_eq!(value, data);
-
-                println_test_status!(tracker, blue, "Test - RAM End 8");
-                // Write last byte
-                core.write_word_8(ram_start + ram_size - 1, 0xcd)
-                    .with_context(|| {
-                        format!("Write_word_8 to address {:08x}", ram_start + ram_size - 1)
-                    })?;
-
-                let value = core
-                    .read_word_8(ram_start + ram_size - 1)
-                    .with_context(|| {
-                        format!("read_word_8 from address {:08x}", ram_start + ram_size - 1)
-                    })?;
-                assert_eq!(value, 0xcd);
-            }
-            // Ignore other types of regions
-            _other => {}
-        }
+    for ram in memory_regions {
+        let ram_start = ram.range.start;
+        let ram_size = ram.range.end - ram.range.start;
+
+        println_test_status!(tracker, blue, "Test - RAM Start 32");
+        // Write first word
+        core.write_word_32(ram_start, 0xababab)?;
+        let value = core.read_word_32(ram_start).into_diagnostic()?;
+        assert_eq!(value, 0xababab);
+
+        println_test_status!(tracker, blue, "Test - RAM End 32");
+        // Write last word
+        core.write_word_32(ram_start + ram_size - 4, 0xababac)
+            .into_diagnostic()?;
+        let value = core
+            .read_word_32(ram_start + ram_size - 4)
+            .into_diagnostic()?;
+        assert_eq!(value, 0xababac);
+
+        println_test_status!(tracker, blue, "Test - RAM Start 8");
+        // Write first byte
+        core.write_word_8(ram_start, 0xac).into_diagnostic()?;
+        let value = core.read_word_8(ram_start).into_diagnostic()?;
+        assert_eq!(value, 0xac);
+
+        println_test_status!(tracker, blue, "Test - RAM 8 Unaligned");
+        let address = ram_start + 1;
+        let data = 0x23;
+        // Write last byte
+        core.write_word_8(address, data)
+            .into_diagnostic()
+            .wrap_err_with(|| format!("Write_word_8 to address {address:08x}"))?;
+
+        let value = core
+            .read_word_8(address)
+            .into_diagnostic()
+            .wrap_err_with(|| format!("read_word_8 from address {address:08x}"))?;
+        assert_eq!(value, data);
+
+        println_test_status!(tracker, blue, "Test - RAM End 8");
+        // Write last byte
+        core.write_word_8(ram_start + ram_size - 1, 0xcd)
+            .into_diagnostic()
+            .wrap_err_with(|| {
+                format!("Write_word_8 to address {:08x}", ram_start + ram_size - 1)
+            })?;
+
+        let value = core
+            .read_word_8(ram_start + ram_size - 1)
+            .into_diagnostic()
+            .wrap_err_with(|| {
+                format!("read_word_8 from address {:08x}", ram_start + ram_size - 1)
+            })?;
+        assert_eq!(value, 0xcd);
     }
 
     Ok(())
 }
 
 #[distributed_slice(CORE_TESTS)]
-fn test_hw_breakpoints(tracker: &TestTracker, core: &mut Core) -> Result<(), probe_rs::Error> {
+fn test_hw_breakpoints(tracker: &TestTracker, core: &mut Core) -> TestResult {
     println_test_status!(tracker, blue, "Testing HW breakpoints");
 
-    let memory_regions: Vec<_> = core.memory_regions().cloned().collect();
+    let memory_regions: Vec<_> = core
+        .memory_regions()
+        .filter_map(|region| match region {
+            MemoryRegion::Nvm(region) => Some(region),
+            _ => None,
+        })
+        .cloned()
+        .collect();
+
+    if memory_regions.is_empty() {
+        return Err(TestFailure::Skipped(
+            "No NVM memory regions found, unable to test HW breakpoints.".to_string(),
+        ));
+    }
 
     // For this test, we assume that code is executed from Flash / non-volatile memory, and try to set breakpoints
     // in these regions.
     for region in memory_regions {
-        match region {
-            probe_rs::config::MemoryRegion::Nvm(nvm) => {
-                let initial_breakpoint_addr = nvm.range.start;
+        let initial_breakpoint_addr = region.range.start;
 
-                let num_breakpoints = core.available_breakpoint_units()?;
+        let num_breakpoints = core.available_breakpoint_units().into_diagnostic()?;
 
-                println_test_status!(tracker, blue, "{} breakpoints supported", num_breakpoints);
+        println_test_status!(tracker, blue, "{} breakpoints supported", num_breakpoints);
 
-                for i in 0..num_breakpoints {
-                    core.set_hw_breakpoint(initial_breakpoint_addr + 4 * i as u64)?;
-                }
-
-                // Try to set an additional breakpoint, which should fail
-                core.set_hw_breakpoint(initial_breakpoint_addr + num_breakpoints as u64 * 4)
-                    .expect_err(
-                        "Trying to use more than supported number of breakpoints should fail.",
-                    );
+        for i in 0..num_breakpoints {
+            core.set_hw_breakpoint(initial_breakpoint_addr + 4 * i as u64)
+                .into_diagnostic()?;
+        }
 
-                // Clear all breakpoints again
-                for i in 0..num_breakpoints {
-                    core.clear_hw_breakpoint(initial_breakpoint_addr + 4 * i as u64)?;
-                }
-            }
+        // Try to set an additional breakpoint, which should fail
+        core.set_hw_breakpoint(initial_breakpoint_addr + num_breakpoints as u64 * 4)
+            .expect_err("Trying to use more than supported number of breakpoints should fail.");
 
-            // Skip other regions
-            _other => {}
+        // Clear all breakpoints again
+        for i in 0..num_breakpoints {
+            core.clear_hw_breakpoint(initial_breakpoint_addr + 4 * i as u64)
+                .into_diagnostic()?;
         }
     }
 
     Ok(())
 }
 
-pub fn test_flashing(
-    tracker: &TestTracker,
-    session: &mut Session,
-    test_binary: &Path,
-) -> Result<()> {
+#[distributed_slice(SESSION_TESTS)]
+pub fn test_flashing(tracker: &TestTracker, session: &mut Session) -> Result<(), TestFailure> {
+    let Some(test_binary) = tracker
+        .current_dut_definition()
+        .flash_test_binary
+        .as_deref()
+    else {
+        return Err(TestFailure::MissingResource(
+            "No flash test binary specified".to_string(),
+        ));
+    };
+
     let progress = FlashProgress::new(|event| {
         log::debug!("Flash Event: {:?}", event);
         eprint!(".");
@@ -191,7 +215,8 @@ pub fn test_flashing(
 
     let start_time = Instant::now();
 
-    download_file_with_options(session, test_binary, Format::Elf, options)?;
+    download_file_with_options(session, test_binary, Format::Elf, options)
+        .map_err(|err| TestFailure::Error(Box::new(err)))?;
 
     println!();
 
diff --git a/smoke-tester/src/tests/stepping.rs b/smoke-tester/src/tests/stepping.rs
index 05a2441c..3c34d05c 100644
--- a/smoke-tester/src/tests/stepping.rs
+++ b/smoke-tester/src/tests/stepping.rs
@@ -1,23 +1,26 @@
 use std::time::Duration;
 
-use anyhow::Result;
 use linkme::distributed_slice;
+use miette::IntoDiagnostic;
 use probe_rs::{
     config::MemoryRegion, probe::DebugProbeError, Architecture, BreakpointCause, Core, CoreStatus,
     Error, HaltReason, MemoryInterface,
 };
 
-use crate::{TestTracker, CORE_TESTS};
+use crate::{TestFailure, TestResult, TestTracker, CORE_TESTS};
 
 const TEST_CODE: &[u8] = include_bytes!("test_arm.bin");
 
 #[distributed_slice(CORE_TESTS)]
-fn test_stepping(_tracker: &TestTracker, core: &mut Core) -> Result<(), probe_rs::Error> {
+fn test_stepping(tracker: &TestTracker, core: &mut Core) -> TestResult {
     println!("Testing stepping...");
 
     if core.architecture() == Architecture::Riscv {
         // Not implemented for RISC-V yet
-        return Ok(());
+        return Err(TestFailure::UnimplementedForTarget(
+            Box::new(tracker.current_target().clone()),
+            "Testing stepping is not implemented for RISC-V yet.".to_string(),
+        ));
     }
 
     let ram_region = core.memory_regions().find_map(MemoryRegion::as_ram_region);
@@ -25,25 +28,28 @@ fn test_stepping(_tracker: &TestTracker, core: &mut Core) -> Result<(), probe_rs
     let ram_region = if let Some(ram_region) = ram_region {
         ram_region.clone()
     } else {
-        return Err(probe_rs::Error::Other(anyhow::anyhow!(
-            "No RAM configured for core!"
-        )));
+        return Err(TestFailure::Skipped(
+            "No RAM configured for core, unable to test stepping".to_string(),
+        ));
     };
 
-    core.reset_and_halt(Duration::from_millis(100))?;
+    core.reset_and_halt(Duration::from_millis(100))
+        .into_diagnostic()?;
 
     let code_load_address = ram_region.range.start;
 
-    core.write_8(code_load_address, TEST_CODE)?;
+    core.write_8(code_load_address, TEST_CODE)
+        .into_diagnostic()?;
 
     let registers = core.registers();
-    core.write_core_reg(registers.pc().unwrap(), code_load_address)?;
+    core.write_core_reg(registers.pc().unwrap(), code_load_address)
+        .into_diagnostic()?;
 
-    let core_information = core.step()?;
+    let core_information = core.step().into_diagnostic()?;
 
     let expected_pc = code_load_address + 2;
 
-    let core_status = core.status()?;
+    let core_status = core.status().into_diagnostic()?;
 
     assert_eq!(
         core_information.pc, expected_pc,
@@ -54,7 +60,9 @@ fn test_stepping(_tracker: &TestTracker, core: &mut Core) -> Result<(), probe_rs
         log::warn!("Unexpected core status: {:?}!", core_status);
     }
 
-    let r0_value: u64 = core.read_core_reg(registers.core_register(0))?;
+    let r0_value: u64 = core
+        .read_core_reg(registers.core_register(0))
+        .into_diagnostic()?;
 
     assert_eq!(r0_value, 0);
 
@@ -67,28 +75,32 @@ fn test_stepping(_tracker: &TestTracker, core: &mut Core) -> Result<(), probe_rs
 
     // Run up to the software breakpoint (bkpt) at offset 0x6
     let break_address = code_load_address + 0x6;
-    core.run()?;
+    core.run().into_diagnostic()?;
 
     match core.wait_for_core_halted(Duration::from_millis(100)) {
         Ok(()) => {}
         Err(Error::Probe(DebugProbeError::Timeout)) => {
             println!("Core did not halt after timeout!");
-            core.halt(Duration::from_millis(100))?;
+            core.halt(Duration::from_millis(100)).into_diagnostic()?;
 
-            let pc: u64 = core.read_core_reg(core.program_counter())?;
+            let pc: u64 = core
+                .read_core_reg(core.program_counter())
+                .into_diagnostic()?;
 
             println!("Core stopped at: {pc:#08x}");
 
-            let r2_val: u64 = core.read_core_reg(registers.core_register(2))?;
+            let r2_val: u64 = core
+                .read_core_reg(registers.core_register(2))
+                .into_diagnostic()?;
 
             println!("$r2 = {r2_val:#08x}");
         }
-        Err(other) => return Err(other),
+        Err(other) => return Err(other.into()),
     }
 
     println!("Core halted again!");
 
-    let core_status = core.status()?;
+    let core_status = core.status().into_diagnostic()?;
 
     assert!(matches!(
         core_status,
@@ -96,19 +108,22 @@ fn test_stepping(_tracker: &TestTracker, core: &mut Core) -> Result<(), probe_rs
             | CoreStatus::Halted(HaltReason::Breakpoint(BreakpointCause::Unknown))
     ));
 
-    let pc: u64 = core.read_core_reg(core.program_counter())?;
+    let pc: u64 = core
+        .read_core_reg(core.program_counter())
+        .into_diagnostic()?;
 
     assert_eq!(pc, break_address);
 
     println!("Core halted at {pc:#08x}, now trying to run...");
 
     // Increase PC by 2 to skip breakpoint.
-    core.write_core_reg(core.program_counter(), pc + 2)?;
+    core.write_core_reg(core.program_counter(), pc + 2)
+        .into_diagnostic()?;
 
     println!("Run core again, with pc = {:#010x}", pc + 2);
 
     // Run to the finish
-    core.run()?;
+    core.run().into_diagnostic()?;
 
     // Final breakpoint is at offset 0x10
 
@@ -118,20 +133,24 @@ fn test_stepping(_tracker: &TestTracker, core: &mut Core) -> Result<(), probe_rs
         Ok(()) => {}
         Err(Error::Probe(DebugProbeError::Timeout)) => {
             println!("Core did not halt after timeout!");
-            core.halt(Duration::from_millis(100))?;
+            core.halt(Duration::from_millis(100)).into_diagnostic()?;
 
-            let pc: u64 = core.read_core_reg(core.program_counter())?;
+            let pc: u64 = core
+                .read_core_reg(core.program_counter())
+                .into_diagnostic()?;
 
             println!("Core stopped at: {pc:#08x}");
 
-            let r2_val: u64 = core.read_core_reg(registers.core_register(2))?;
+            let r2_val: u64 = core
+                .read_core_reg(registers.core_register(2))
+                .into_diagnostic()?;
 
             println!("$r2 = {r2_val:#08x}");
         }
-        Err(other) => return Err(other),
+        Err(other) => return Err(other.into()),
     }
 
-    let core_status = core.status()?;
+    let core_status = core.status().into_diagnostic()?;
 
     assert!(matches!(
         core_status,
@@ -139,12 +158,16 @@ fn test_stepping(_tracker: &TestTracker, core: &mut Core) -> Result<(), probe_rs
             | CoreStatus::Halted(HaltReason::Breakpoint(BreakpointCause::Unknown))
     ));
 
-    let pc: u64 = core.read_core_reg(core.program_counter())?;
+    let pc: u64 = core
+        .read_core_reg(core.program_counter())
+        .into_diagnostic()?;
 
     assert_eq!(pc, break_address, "{pc:#08x} != {break_address:#08x}");
 
     // Register r2 should be 1 to indicate end of test.
-    let r2_val: u64 = core.read_core_reg(registers.core_register(2))?;
+    let r2_val: u64 = core
+        .read_core_reg(registers.core_register(2))
+        .into_diagnostic()?;
     assert_eq!(1, r2_val);
 
     Ok(())