Repository: chipsalliance/caliptra-sw
Pull request #126 (2023-03-30 23:36:00)

Add integration tests for runtime firmware
First commit moves the driver test harness to its own crate.

Second commit adds the ability to test two scenarios:

* Boot ROM + FMC, then run a self-contained test runtime image
* Boot ROM, FMC, and Runtime, then run end-to-end integration tests via
  soc registers and the mailbox.

korran: Looks to be related to the emu feature. This seems to help:

```
diff --git a/drivers/test-fw/Cargo.toml b/drivers/test-fw/Cargo.toml
index 4fd5499..1d86b51 100644
--- a/drivers/test-fw/Cargo.toml
+++ b/drivers/test-fw/Cargo.toml
@@ -14,7 +14,7 @@ caliptra-test-harness = { path = "../../test-harness" }
 cfg-if = "1.0.0"
 
 [features]
-emu = []
+emu = ["caliptra-test-harness/emu"]
 
 # This feature is used to filter all these binary targets during normal builds
 # (targets must be built with cargo arguments:
diff --git a/test-harness/Cargo.toml b/test-harness/Cargo.toml
index d7963e3..177d70b 100644
--- a/test-harness/Cargo.toml
+++ b/test-harness/Cargo.toml
@@ -13,3 +13,4 @@ runtime = []
 
 [dependencies]
 cfg-if = "1.0.0"
+caliptra-drivers = { path = "../drivers", features=["emu"] }
```

Patch:
diff --git a/Cargo.lock b/Cargo.lock
index 7f01465d..21a9ddd0 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -149,6 +149,7 @@ dependencies = [
  "caliptra-drivers",
  "caliptra-kat",
  "caliptra-registers",
+ "caliptra-test-harness",
  "cfg-if",
 ]
 
@@ -419,12 +420,29 @@ dependencies = [
  "caliptra-builder",
  "caliptra-cpu",
  "caliptra-drivers",
+ "caliptra-hw-model",
+ "caliptra-image-elf",
+ "caliptra-image-fake-keys",
+ "caliptra-image-gen",
+ "caliptra-image-openssl",
+ "caliptra-image-serde",
+ "caliptra-image-types",
  "caliptra_common",
  "cfg-if",
  "ufmt",
  "zerocopy",
 ]
 
+[[package]]
+name = "caliptra-runtime-test-bin"
+version = "0.1.0"
+dependencies = [
+ "caliptra-cpu",
+ "caliptra-drivers",
+ "caliptra-test-harness",
+ "cfg-if",
+]
+
 [[package]]
 name = "caliptra-systemrdl"
 version = "0.1.0"
@@ -437,6 +455,14 @@ dependencies = [
  "caliptra-hw-model",
 ]
 
+[[package]]
+name = "caliptra-test-harness"
+version = "0.1.0"
+dependencies = [
+ "caliptra-drivers",
+ "cfg-if",
+]
+
 [[package]]
 name = "caliptra-verilated"
 version = "0.1.0"
diff --git a/Cargo.toml b/Cargo.toml
index e65f0591..4b2cf7c3 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -27,6 +27,7 @@ members = [
   "registers",
   "registers/bin/generator",
   "runtime",
+  "runtime/test-fw",
   "sw-emulator/app",
   "sw-emulator/compliance-test",
   "sw-emulator/lib/cpu",
@@ -52,6 +53,7 @@ members = [
   "rom/dev/tools/test-fmc",
   "rom/dev/tools/test-rt",
   "test",
+  "test-harness",
 ]
 
 [profile.firmware]
diff --git a/runtime/Cargo.toml b/runtime/Cargo.toml
index f7ba48a3..75202ac9 100644
--- a/runtime/Cargo.toml
+++ b/runtime/Cargo.toml
@@ -16,7 +16,14 @@ zerocopy = "0.6.1"
 cfg-if = "1.0.0"
 
 [dev-dependencies]
+caliptra-hw-model = { path = "../hw-model" }
 caliptra-builder = { path = "../builder" }
+caliptra-image-elf = { path = "../image/elf" }
+caliptra-image-fake-keys = { path = "../image/fake-keys" }
+caliptra-image-gen = { path = "../image/gen" }
+caliptra-image-openssl = { path = "../image/openssl" }
+caliptra-image-serde = { path = "../image/serde" }
+caliptra-image-types = { path = "../image/types" }
 
 [features]
 riscv = ["caliptra-cpu/riscv"]
diff --git a/runtime/src/main.rs b/runtime/src/main.rs
index 7b53cfb7..e5d3c162 100644
--- a/runtime/src/main.rs
+++ b/runtime/src/main.rs
@@ -86,7 +86,7 @@ extern "C" fn nmi_handler(trap_record: &TrapRecord) {
 #[inline(never)]
 #[cfg(not(feature = "std"))]
 #[allow(clippy::empty_loop)]
-fn fmc_panic(_: &core::panic::PanicInfo) -> ! {
+fn runtime_panic(_: &core::panic::PanicInfo) -> ! {
     cprintln!("RT Panic!!");
     panic_is_possible();
 


Test patch:
diff --git a/drivers/test-fw/Cargo.toml b/drivers/test-fw/Cargo.toml
index 69d9b138..ea1be866 100644
--- a/drivers/test-fw/Cargo.toml
+++ b/drivers/test-fw/Cargo.toml
@@ -10,17 +10,18 @@ edition = "2021"
 caliptra-drivers = { path = "..", features=["emu"] }
 caliptra-registers = { path = "../../registers" }
 caliptra-kat = { path = "../../kat" }
+caliptra-test-harness = { path = "../../test-harness" }
 cfg-if = "1.0.0"
 
 [features]
-emu = []
+emu = ["caliptra-test-harness/emu"]
 
 # This feature is used to filter all these binary targets during normal builds
 # (targets must be built with cargo arguments:
 #     --target riscv32imc-unknown-none-elf \
 #     --profile=firmware \
 #     --features riscv
-riscv = []
+riscv = ["caliptra-test-harness/riscv"]
 
 [[bin]]
 name = "sha1"
diff --git a/drivers/test-fw/src/bin/doe_tests.rs b/drivers/test-fw/src/bin/doe_tests.rs
index 9754f213..5c270ffb 100644
--- a/drivers/test-fw/src/bin/doe_tests.rs
+++ b/drivers/test-fw/src/bin/doe_tests.rs
@@ -17,7 +17,7 @@ Abstract:
 
 use caliptra_drivers::{Array4x4, DeobfuscationEngine, KeyId};
 
-mod harness;
+use caliptra_test_harness::test_suite;
 
 fn test_decrypt_uds() {
     let iv = [0xFF_u8; 16];
diff --git a/drivers/test-fw/src/bin/ecc384_tests.rs b/drivers/test-fw/src/bin/ecc384_tests.rs
index bb20bf9e..22170186 100644
--- a/drivers/test-fw/src/bin/ecc384_tests.rs
+++ b/drivers/test-fw/src/bin/ecc384_tests.rs
@@ -21,7 +21,7 @@ use caliptra_drivers::{
 };
 use caliptra_kat::Ecc384Kat;
 
-mod harness;
+use caliptra_test_harness::test_suite;
 
 const PRIV_KEY: [u8; 48] = [
     0xfe, 0xee, 0xf5, 0x54, 0x4a, 0x76, 0x56, 0x49, 0x90, 0x12, 0x8a, 0xd1, 0x89, 0xe8, 0x73, 0xf2,
diff --git a/drivers/test-fw/src/bin/error_reporter_tests.rs b/drivers/test-fw/src/bin/error_reporter_tests.rs
index 54056354..17217b76 100644
--- a/drivers/test-fw/src/bin/error_reporter_tests.rs
+++ b/drivers/test-fw/src/bin/error_reporter_tests.rs
@@ -21,7 +21,7 @@ use caliptra_drivers::{
 };
 use caliptra_registers::soc_ifc;
 
-mod harness;
+use caliptra_test_harness::test_suite;
 
 fn test_report_fw_error() {
     let v: u32 = 0xdead0;
diff --git a/drivers/test-fw/src/bin/hmac384_tests.rs b/drivers/test-fw/src/bin/hmac384_tests.rs
index 10dfd434..6134d214 100755
--- a/drivers/test-fw/src/bin/hmac384_tests.rs
+++ b/drivers/test-fw/src/bin/hmac384_tests.rs
@@ -21,7 +21,7 @@ use caliptra_drivers::{
 };
 use caliptra_kat::Hmac384Kat;
 
-mod harness;
+use caliptra_test_harness::test_suite;
 
 fn test_hmac0() {
     let key: [u8; 48] = [
@@ -225,7 +225,7 @@ fn test_hmac5() {
 /// Generate the HMAC of the output tag in the buffer - step_1 Tag
 ///
 ///
-/// Step 2:  
+/// Step 2:
 /// Key From Key Vault
 /// Generate the output tag that goes in the KV
 /// Generate the HMAC of the tag in KV and the tag goes in specified buffer
diff --git a/drivers/test-fw/src/bin/keyvault_tests.rs b/drivers/test-fw/src/bin/keyvault_tests.rs
index 3acd1c63..4c2d6b58 100644
--- a/drivers/test-fw/src/bin/keyvault_tests.rs
+++ b/drivers/test-fw/src/bin/keyvault_tests.rs
@@ -16,7 +16,7 @@ Abstract:
 #![no_main]
 
 use caliptra_drivers::{KeyId, KeyUsage, KeyVault};
-mod harness;
+use caliptra_test_harness::test_suite;
 
 const KEY_IDS: [KeyId; 32] = [
     KeyId::KeyId0,
diff --git a/drivers/test-fw/src/bin/mailbox_tests.rs b/drivers/test-fw/src/bin/mailbox_tests.rs
index ce4c25c8..e1e5544b 100644
--- a/drivers/test-fw/src/bin/mailbox_tests.rs
+++ b/drivers/test-fw/src/bin/mailbox_tests.rs
@@ -20,7 +20,7 @@ use caliptra_registers::mbox::{self};
 use core::mem::size_of;
 use core::slice;
 
-mod harness;
+use caliptra_test_harness::test_suite;
 
 fn test_send_txn_drop() {
     let mut ii = 0;
diff --git a/drivers/test-fw/src/bin/pcrbank_tests.rs b/drivers/test-fw/src/bin/pcrbank_tests.rs
index 8ca7d2b5..5823c586 100644
--- a/drivers/test-fw/src/bin/pcrbank_tests.rs
+++ b/drivers/test-fw/src/bin/pcrbank_tests.rs
@@ -17,7 +17,7 @@ Abstract:
 
 use caliptra_drivers::{PcrBank, PcrId};
 
-mod harness;
+use caliptra_test_harness::test_suite;
 
 const PCR_IDS: [PcrId; 32] = [
     PcrId::PcrId0,
diff --git a/drivers/test-fw/src/bin/sha1_tests.rs b/drivers/test-fw/src/bin/sha1_tests.rs
index 173446da..e5223e66 100644
--- a/drivers/test-fw/src/bin/sha1_tests.rs
+++ b/drivers/test-fw/src/bin/sha1_tests.rs
@@ -18,7 +18,7 @@ Abstract:
 use caliptra_drivers::{Array4x5, Array4xN, Sha1};
 use caliptra_kat::Sha1Kat;
 
-mod harness;
+use caliptra_test_harness::test_suite;
 
 fn test_sha1(data: &str, expected: Array4x5) {
     let mut digest = Array4x5::default();
diff --git a/drivers/test-fw/src/bin/sha256_tests.rs b/drivers/test-fw/src/bin/sha256_tests.rs
index 176f15a0..e4655e0f 100644
--- a/drivers/test-fw/src/bin/sha256_tests.rs
+++ b/drivers/test-fw/src/bin/sha256_tests.rs
@@ -18,7 +18,7 @@ Abstract:
 use caliptra_drivers::{Array4x8, Sha256};
 use caliptra_kat::Sha256Kat;
 
-mod harness;
+use caliptra_test_harness::test_suite;
 
 fn test_digest0() {
     let expected: [u8; 32] = [
diff --git a/drivers/test-fw/src/bin/sha384_tests.rs b/drivers/test-fw/src/bin/sha384_tests.rs
index 1364226e..51261160 100644
--- a/drivers/test-fw/src/bin/sha384_tests.rs
+++ b/drivers/test-fw/src/bin/sha384_tests.rs
@@ -18,7 +18,7 @@ Abstract:
 use caliptra_drivers::{Array4x12, PcrBank, PcrId, Sha384};
 use caliptra_kat::Sha384Kat;
 
-mod harness;
+use caliptra_test_harness::test_suite;
 
 fn test_digest0() {
     let expected: [u8; 48] = [
diff --git a/drivers/test-fw/src/bin/sha384acc_tests.rs b/drivers/test-fw/src/bin/sha384acc_tests.rs
index 4a10ee08..3ef77996 100644
--- a/drivers/test-fw/src/bin/sha384acc_tests.rs
+++ b/drivers/test-fw/src/bin/sha384acc_tests.rs
@@ -17,7 +17,7 @@ Abstract:
 
 use caliptra_drivers::{Array4x12, Mailbox, Sha384Acc};
 use caliptra_kat::Sha384AccKat;
-mod harness;
+use caliptra_test_harness::test_suite;
 
 const MAX_MAILBOX_CAPACITY_BYTES: usize = 128 << 10;
 const SHA384_HASH_SIZE: usize = 48;
diff --git a/drivers/test-fw/src/bin/status_reporter_tests.rs b/drivers/test-fw/src/bin/status_reporter_tests.rs
index 01a0cf88..b731f99d 100644
--- a/drivers/test-fw/src/bin/status_reporter_tests.rs
+++ b/drivers/test-fw/src/bin/status_reporter_tests.rs
@@ -18,7 +18,7 @@ Abstract:
 use caliptra_drivers::{report_boot_status, FlowStatus};
 use caliptra_registers::soc_ifc;
 
-mod harness;
+use caliptra_test_harness::test_suite;
 
 fn retrieve_boot_status() -> u32 {
     let soc_ifc = soc_ifc::RegisterBlock::soc_ifc_reg();
diff --git a/runtime/test-fw/Cargo.toml b/runtime/test-fw/Cargo.toml
new file mode 100644
index 00000000..fe0f84aa
--- /dev/null
+++ b/runtime/test-fw/Cargo.toml
@@ -0,0 +1,27 @@
+# Licensed under the Apache-2.0 license
+
+[package]
+name = "caliptra-runtime-test-bin"
+version = "0.1.0"
+edition = "2021"
+
+[features]
+runtime = ["caliptra-test-harness/runtime"]
+riscv = ["caliptra-cpu/riscv", "runtime"]
+emu = [
+    "caliptra-drivers/emu"
+]
+
+[[bin]]
+name = "boot"
+path = "src/boot_tests.rs"
+required-features = ["riscv"]
+
+[build-dependencies]
+cfg-if = "1.0.0"
+
+[dependencies]
+caliptra-test-harness = { path = "../../test-harness" }
+caliptra-cpu = { version = "0.1.0", path = "../../cpu" }
+caliptra-drivers = { path = "../../drivers" }
+cfg-if = "1.0.0"
diff --git a/runtime/test-fw/build.rs b/runtime/test-fw/build.rs
new file mode 100644
index 00000000..740b4e08
--- /dev/null
+++ b/runtime/test-fw/build.rs
@@ -0,0 +1,23 @@
+// Licensed under the Apache-2.0 license
+
+fn main() {
+    cfg_if::cfg_if! {
+        if #[cfg(not(feature = "std"))] {
+            use std::env;
+            use std::fs;
+            use std::path::PathBuf;
+
+            let out_dir = PathBuf::from(env::var("OUT_DIR").unwrap());
+
+            // Put the linker script somewhere the linker can find it.
+            fs::write(out_dir.join("memory.x"), include_bytes!("../memory.x")).unwrap();
+            println!("cargo:rustc-link-search={}", out_dir.display());
+
+            println!("cargo:rerun-if-changed=memory.x");
+            println!("cargo:rustc-link-arg=-Tmemory.x");
+
+            println!("cargo:rustc-link-arg=-Tlink.x");
+            println!("cargo:rerun-if-changed=build.rs");
+        }
+    }
+}
diff --git a/runtime/test-fw/src/boot_tests.rs b/runtime/test-fw/src/boot_tests.rs
new file mode 100644
index 00000000..6b4ca7a1
--- /dev/null
+++ b/runtime/test-fw/src/boot_tests.rs
@@ -0,0 +1,26 @@
+/*++
+
+Licensed under the Apache-2.0 license.
+
+File Name:
+
+    boot_tests.rs
+
+Abstract:
+
+    File contains test cases for booting runtime firmware
+
+--*/
+
+#![no_std]
+#![no_main]
+
+use caliptra_test_harness::{runtime_handlers, test_suite};
+
+fn test_boot() {
+    assert!(true);
+}
+
+test_suite! {
+    test_boot,
+}
diff --git a/runtime/tests/integration_tests.rs b/runtime/tests/integration_tests.rs
new file mode 100644
index 00000000..a5fe1878
--- /dev/null
+++ b/runtime/tests/integration_tests.rs
@@ -0,0 +1,47 @@
+// Licensed under the Apache-2.0 license.
+
+use caliptra_builder::{FwId, ImageOptions, APP_WITH_UART, FMC_WITH_UART, ROM_WITH_UART};
+use caliptra_hw_model::{BootParams, HwModel, InitParams};
+
+fn run_rt_test(test_bin_name: Option<&str>) {
+    let runtime_fwid = match test_bin_name {
+        Some(bin) => FwId {
+            crate_name: "caliptra-runtime-test-bin",
+            bin_name: bin,
+            features: &["emu", "riscv", "runtime"],
+        },
+        None => APP_WITH_UART,
+    };
+
+    let rom = caliptra_builder::build_firmware_rom(&ROM_WITH_UART).unwrap();
+    let image = caliptra_builder::build_and_sign_image(
+        &FMC_WITH_UART,
+        &runtime_fwid,
+        ImageOptions::default(),
+    )
+    .unwrap();
+
+    let mut model = caliptra_hw_model::new(BootParams {
+        init_params: InitParams {
+            rom: &rom,
+            ..Default::default()
+        },
+        ..Default::default()
+    })
+    .unwrap();
+
+    model.upload_firmware(&image).unwrap();
+}
+
+#[test]
+fn test_standard() {
+    // Test that the normal runtime firmware boots.
+    // Ultimately, this will be useful for exercising Caliptra end-to-end
+    // via the mailbox.
+    run_rt_test(None);
+}
+
+#[test]
+fn test_boot() {
+    run_rt_test(Some("boot"));
+}
diff --git a/test-harness/Cargo.toml b/test-harness/Cargo.toml
new file mode 100644
index 00000000..2473b308
--- /dev/null
+++ b/test-harness/Cargo.toml
@@ -0,0 +1,17 @@
+# Licensed under the Apache-2.0 license
+
+[package]
+name = "caliptra-test-harness"
+version = "0.1.0"
+edition = "2021"
+
+[features]
+emu = []
+riscv = []
+
+# Whether or not this is a harness for a runtime firmware image
+runtime = []
+
+[dependencies]
+cfg-if = "1.0.0"
+caliptra-drivers = { path = "../drivers", features=["emu"] }
diff --git a/drivers/test-fw/src/bin/harness/mod.rs b/test-harness/src/lib.rs
similarity index 53%
rename from drivers/test-fw/src/bin/harness/mod.rs
rename to test-harness/src/lib.rs
index 10a138f7..2b412c42 100644
--- a/drivers/test-fw/src/bin/harness/mod.rs
+++ b/test-harness/src/lib.rs
@@ -14,12 +14,19 @@ References:
     https://os.phil-opp.com/vga-text-mode for print functionality.
 
 --*/
+#![no_std]
 
 use core::fmt;
+use core::format_args;
+use core::ops::Fn;
+
+// If not using the runtime entrypoint, include a test start.S
+#[cfg(all(feature = "riscv", not(feature = "runtime")))]
+core::arch::global_asm!(include_str!("start.S"));
 
 #[macro_export]
 macro_rules! print {
-    ($($arg:tt)*) => ($crate::harness::_print(format_args!($($arg)*)));
+    ($($arg:tt)*) => ($crate::_print(format_args!($($arg)*)));
 }
 
 #[macro_export]
@@ -42,14 +49,44 @@ pub fn _print(args: fmt::Arguments) {
     }
 }
 
+#[macro_export]
+macro_rules! runtime_handlers {
+    () => {
+        use caliptra_cpu::TrapRecord;
+
+        #[no_mangle]
+        #[inline(never)]
+        extern "C" fn exception_handler(trap_record: &TrapRecord) {
+            println!(
+                "TEST EXCEPTION mcause=0x{:08X} mscause=0x{:08X} mepc=0x{:08X}",
+                trap_record.mcause, trap_record.mscause, trap_record.mepc
+            );
+
+            // Signal non-fatal error to SOC
+            caliptra_drivers::report_fw_error_non_fatal(0xdead0);
+
+            assert!(false);
+        }
+
+        #[no_mangle]
+        #[inline(never)]
+        extern "C" fn nmi_handler(trap_record: &TrapRecord) {
+            println!(
+                "TEST NMI mcause=0x{:08X} mscause=0x{:08X} mepc=0x{:08X}",
+                trap_record.mcause, trap_record.mscause, trap_record.mepc
+            );
+
+            assert!(false);
+        }
+    };
+}
+
 #[macro_export]
 macro_rules! test_suite {
     ($($test_case: ident,)*) => {
         use core::arch::global_asm;
         use core::panic::PanicInfo;
-        use $crate::harness::Testable;
-
-        global_asm!(include_str!("start.S"));
+        use caliptra_test_harness::{println, Testable};
 
         #[panic_handler]
         pub fn panic(info: &PanicInfo) -> ! {
@@ -71,6 +108,15 @@ macro_rules! test_suite {
                 $test_case.run();
             )*
         }
+
+        #[no_mangle]
+        pub extern "C" fn entry_point() {
+            main();
+            caliptra_drivers::ExitCtrl::exit(0);
+        }
+
+        #[cfg(feature = "runtime")]
+        runtime_handlers! {}
     };
 }