Repository: foundry-rs/foundry (language: rust)

Issue:
feat(fuzz): generate solidity regression tests from failures
### Component

Forge

### Describe the feature you would like

Complementary to persist and replay fuzz / invariant failures, based on same data we can also generate Solidity regression tests (configs TBD, on demand, configurable dir output)

### Additional context

_No response_


Discussion / hints:
That would be amazing!

Fix patch:
diff --git a/crates/config/src/invariant.rs b/crates/config/src/invariant.rs
index 003b1bac9365..3fb3b8f7e1d4 100644
--- a/crates/config/src/invariant.rs
+++ b/crates/config/src/invariant.rs
@@ -32,6 +32,8 @@ pub struct InvariantConfig {
     pub show_metrics: bool,
     /// Optional timeout (in seconds) for each invariant test.
     pub timeout: Option<u32>,
+    /// Display counterexample as solidity calls.
+    pub show_solidity: bool,
 }
 
 impl Default for InvariantConfig {
@@ -48,6 +50,7 @@ impl Default for InvariantConfig {
             failure_persist_dir: None,
             show_metrics: false,
             timeout: None,
+            show_solidity: false,
         }
     }
 }
@@ -67,6 +70,7 @@ impl InvariantConfig {
             failure_persist_dir: Some(cache_dir),
             show_metrics: false,
             timeout: None,
+            show_solidity: false,
         }
     }
 
diff --git a/crates/evm/evm/src/executors/invariant/replay.rs b/crates/evm/evm/src/executors/invariant/replay.rs
index 36192a6d6914..24897f8e5fba 100644
--- a/crates/evm/evm/src/executors/invariant/replay.rs
+++ b/crates/evm/evm/src/executors/invariant/replay.rs
@@ -32,6 +32,7 @@ pub fn replay_run(
     coverage: &mut Option<HitMaps>,
     deprecated_cheatcodes: &mut HashMap<&'static str, Option<&'static str>>,
     inputs: &[BasicTxDetails],
+    show_solidity: bool,
 ) -> Result<Vec<BaseCounterExample>> {
     // We want traces for a failed case.
     if executor.inspector().tracer.is_none() {
@@ -64,6 +65,7 @@ pub fn replay_run(
             &tx.call_details.calldata,
             &ided_contracts,
             call_result.traces,
+            show_solidity,
         ));
     }
 
@@ -110,6 +112,7 @@ pub fn replay_error(
     coverage: &mut Option<HitMaps>,
     deprecated_cheatcodes: &mut HashMap<&'static str, Option<&'static str>>,
     progress: Option<&ProgressBar>,
+    show_solidity: bool,
 ) -> Result<Vec<BaseCounterExample>> {
     match failed_case.test_error {
         // Don't use at the moment.
@@ -137,6 +140,7 @@ pub fn replay_error(
                 coverage,
                 deprecated_cheatcodes,
                 &calls,
+                show_solidity,
             )
         }
     }
diff --git a/crates/evm/fuzz/src/lib.rs b/crates/evm/fuzz/src/lib.rs
index c1854f55c1e4..65ef76f16c98 100644
--- a/crates/evm/fuzz/src/lib.rs
+++ b/crates/evm/fuzz/src/lib.rs
@@ -41,21 +41,28 @@ pub enum CounterExample {
 
 #[derive(Clone, Debug, Serialize, Deserialize)]
 pub struct BaseCounterExample {
-    /// Address which makes the call
+    /// Address which makes the call.
     pub sender: Option<Address>,
-    /// Address to which to call to
+    /// Address to which to call to.
     pub addr: Option<Address>,
-    /// The data to provide
+    /// The data to provide.
     pub calldata: Bytes,
-    /// Contract name if it exists
+    /// Contract name if it exists.
     pub contract_name: Option<String>,
-    /// Function signature if it exists
+    /// Function name if it exists.
+    pub func_name: Option<String>,
+    /// Function signature if it exists.
     pub signature: Option<String>,
-    /// Args used to call the function
+    /// Pretty formatted args used to call the function.
     pub args: Option<String>,
-    /// Traces
+    /// Unformatted args used to call the function.
+    pub raw_args: Option<String>,
+    /// Counter example traces.
     #[serde(skip)]
     pub traces: Option<SparsedTraceArena>,
+    /// Whether to display sequence as solidity.
+    #[serde(skip)]
+    pub show_solidity: bool,
 }
 
 impl BaseCounterExample {
@@ -66,6 +73,7 @@ impl BaseCounterExample {
         bytes: &Bytes,
         contracts: &ContractsByAddress,
         traces: Option<SparsedTraceArena>,
+        show_solidity: bool,
     ) -> Self {
         if let Some((name, abi)) = &contracts.get(&addr) {
             if let Some(func) = abi.functions().find(|f| f.selector() == bytes[..4]) {
@@ -76,11 +84,16 @@ impl BaseCounterExample {
                         addr: Some(addr),
                         calldata: bytes.clone(),
                         contract_name: Some(name.clone()),
+                        func_name: Some(func.name.clone()),
                         signature: Some(func.signature()),
                         args: Some(
                             foundry_common::fmt::format_tokens(&args).format(", ").to_string(),
                         ),
+                        raw_args: Some(
+                            foundry_common::fmt::format_tokens_raw(&args).format(", ").to_string(),
+                        ),
                         traces,
+                        show_solidity,
                     };
                 }
             }
@@ -91,9 +104,12 @@ impl BaseCounterExample {
             addr: Some(addr),
             calldata: bytes.clone(),
             contract_name: None,
+            func_name: None,
             signature: None,
             args: None,
+            raw_args: None,
             traces,
+            show_solidity: false,
         }
     }
 
@@ -108,17 +124,40 @@ impl BaseCounterExample {
             addr: None,
             calldata: bytes,
             contract_name: None,
+            func_name: None,
             signature: None,
             args: Some(foundry_common::fmt::format_tokens(&args).format(", ").to_string()),
+            raw_args: Some(foundry_common::fmt::format_tokens_raw(&args).format(", ").to_string()),
             traces,
+            show_solidity: false,
         }
     }
 }
 
 impl fmt::Display for BaseCounterExample {
     fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        // Display counterexample as solidity.
+        if self.show_solidity {
+            if let (Some(sender), Some(contract), Some(address), Some(func_name), Some(args)) =
+                (&self.sender, &self.contract_name, &self.addr, &self.func_name, &self.raw_args)
+            {
+                writeln!(f, "\t\tvm.prank({sender});")?;
+                write!(
+                    f,
+                    "\t\t{}({}).{}({});",
+                    contract.split_once(':').map_or(contract.as_str(), |(_, contract)| contract),
+                    address,
+                    func_name,
+                    args
+                )?;
+
+                return Ok(())
+            }
+        }
+
+        // Regular counterexample display.
         if let Some(sender) = self.sender {
-            write!(f, "sender={sender} addr=")?
+            write!(f, "\t\tsender={sender} addr=")?
         }
 
         if let Some(name) = &self.contract_name {
diff --git a/crates/forge/src/result.rs b/crates/forge/src/result.rs
index 58b0f314599b..0b1ebff19d74 100644
--- a/crates/forge/src/result.rs
+++ b/crates/forge/src/result.rs
@@ -455,7 +455,7 @@ impl fmt::Display for TestResult {
                                     .as_str(),
                                 );
                                 for ex in sequence {
-                                    writeln!(s, "\t\t{ex}").unwrap();
+                                    writeln!(s, "{ex}").unwrap();
                                 }
                             }
                         }
diff --git a/crates/forge/src/runner.rs b/crates/forge/src/runner.rs
index 495b84f3d10c..1b30dcebe6ce 100644
--- a/crates/forge/src/runner.rs
+++ b/crates/forge/src/runner.rs
@@ -581,20 +581,24 @@ impl<'a> FunctionRunner<'a> {
 
         let failure_dir = invariant_config.clone().failure_dir(self.cr.name);
         let failure_file = failure_dir.join(&invariant_contract.invariant_function.name);
+        let show_solidity = invariant_config.clone().show_solidity;
 
         // Try to replay recorded failure if any.
-        if let Ok(call_sequence) =
+        if let Ok(mut call_sequence) =
             foundry_common::fs::read_json_file::<Vec<BaseCounterExample>>(failure_file.as_path())
         {
             // Create calls from failed sequence and check if invariant still broken.
             let txes = call_sequence
-                .iter()
-                .map(|seq| BasicTxDetails {
-                    sender: seq.sender.unwrap_or_default(),
-                    call_details: CallDetails {
-                        target: seq.addr.unwrap_or_default(),
-                        calldata: seq.calldata.clone(),
-                    },
+                .iter_mut()
+                .map(|seq| {
+                    seq.show_solidity = show_solidity;
+                    BasicTxDetails {
+                        sender: seq.sender.unwrap_or_default(),
+                        call_details: CallDetails {
+                            target: seq.addr.unwrap_or_default(),
+                            calldata: seq.calldata.clone(),
+                        },
+                    }
                 })
                 .collect::<Vec<BasicTxDetails>>();
             if let Ok((success, replayed_entirely)) = check_sequence(
@@ -624,6 +628,7 @@ impl<'a> FunctionRunner<'a> {
                         &mut self.result.coverage,
                         &mut self.result.deprecated_cheatcodes,
                         &txes,
+                        show_solidity,
                     );
                     self.result.invariant_replay_fail(
                         replayed_entirely,
@@ -674,6 +679,7 @@ impl<'a> FunctionRunner<'a> {
                         &mut self.result.coverage,
                         &mut self.result.deprecated_cheatcodes,
                         progress.as_ref(),
+                        show_solidity,
                     ) {
                         Ok(call_sequence) => {
                             if !call_sequence.is_empty() {
@@ -719,6 +725,7 @@ impl<'a> FunctionRunner<'a> {
                     &mut self.result.coverage,
                     &mut self.result.deprecated_cheatcodes,
                     &invariant_result.last_run_inputs,
+                    show_solidity,
                 ) {
                     error!(%err, "Failed to replay last invariant run");
                 }


Test patch:
diff --git a/crates/forge/tests/cli/config.rs b/crates/forge/tests/cli/config.rs
index 350a2fed5157..5952da4b38f2 100644
--- a/crates/forge/tests/cli/config.rs
+++ b/crates/forge/tests/cli/config.rs
@@ -1090,6 +1090,7 @@ max_assume_rejects = 65536
 gas_report_samples = 256
 failure_persist_dir = "cache/invariant"
 show_metrics = false
+show_solidity = false
 
 [labels]
 
@@ -1193,7 +1194,8 @@ exclude = []
     "gas_report_samples": 256,
     "failure_persist_dir": "cache/invariant",
     "show_metrics": false,
-    "timeout": null
+    "timeout": null,
+    "show_solidity": false
   },
   "ffi": false,
   "allow_internal_expect_revert": false,
diff --git a/crates/forge/tests/it/invariant.rs b/crates/forge/tests/it/invariant.rs
index fb2a9979f0f3..a88ed3db21c4 100644
--- a/crates/forge/tests/it/invariant.rs
+++ b/crates/forge/tests/it/invariant.rs
@@ -1066,6 +1066,7 @@ contract InvariantSelectorsWeightTest is Test {
 });
 
 // Tests original and new counterexample lengths are displayed on failure.
+// Tests switch from regular sequence output to solidity.
 forgetest_init!(invariant_sequence_len, |prj, cmd| {
     prj.update_config(|config| {
         config.fuzz.seed = Some(U256::from(100u32));
@@ -1099,4 +1100,76 @@ contract InvariantSequenceLenTest is Test {
 	[Sequence] (original: 4, shrunk: 1)
 ...
 "#]]);
+
+    // Check regular sequence output. Shrink disabled to show several lines.
+    cmd.forge_fuse().arg("clean").assert_success();
+    prj.update_config(|config| {
+        config.invariant.shrink_run_limit = 0;
+    });
+    cmd.forge_fuse().args(["test", "--mt", "invariant_increment"]).assert_failure().stdout_eq(
+        str![[r#"
+...
+Failing tests:
+Encountered 1 failing test in test/InvariantSequenceLenTest.t.sol:InvariantSequenceLenTest
+[FAIL: revert: invariant increment failure]
+	[Sequence] (original: 4, shrunk: 4)
+		sender=0x00000000000000000000000000000000000018dE addr=[src/Counter.sol:Counter]0x5615dEB798BB3E4dFa0139dFa1b3D433Cc23b72f calldata=setNumber(uint256) args=[1931387396117645594923 [1.931e21]]
+		sender=0x00000000000000000000000000000000000009d5 addr=[src/Counter.sol:Counter]0x5615dEB798BB3E4dFa0139dFa1b3D433Cc23b72f calldata=increment() args=[]
+		sender=0x0000000000000000000000000000000000000105 addr=[src/Counter.sol:Counter]0x5615dEB798BB3E4dFa0139dFa1b3D433Cc23b72f calldata=increment() args=[]
+		sender=0x00000000000000000000000000000000000009B2 addr=[src/Counter.sol:Counter]0x5615dEB798BB3E4dFa0139dFa1b3D433Cc23b72f calldata=setNumber(uint256) args=[996881781832960761274744263729582347 [9.968e35]]
+ invariant_increment() (runs: 0, calls: 0, reverts: 0)
+
+Encountered a total of 1 failing tests, 0 tests succeeded
+
+"#]],
+    );
+
+    // Check solidity sequence output on same failure.
+    cmd.forge_fuse().arg("clean").assert_success();
+    prj.update_config(|config| {
+        config.invariant.show_solidity = true;
+    });
+    cmd.forge_fuse().args(["test", "--mt", "invariant_increment"]).assert_failure().stdout_eq(
+        str![[r#"
+...
+Failing tests:
+Encountered 1 failing test in test/InvariantSequenceLenTest.t.sol:InvariantSequenceLenTest
+[FAIL: revert: invariant increment failure]
+	[Sequence] (original: 4, shrunk: 4)
+		vm.prank(0x00000000000000000000000000000000000018dE);
+		Counter(0x5615dEB798BB3E4dFa0139dFa1b3D433Cc23b72f).setNumber(1931387396117645594923);
+		vm.prank(0x00000000000000000000000000000000000009d5);
+		Counter(0x5615dEB798BB3E4dFa0139dFa1b3D433Cc23b72f).increment();
+		vm.prank(0x0000000000000000000000000000000000000105);
+		Counter(0x5615dEB798BB3E4dFa0139dFa1b3D433Cc23b72f).increment();
+		vm.prank(0x00000000000000000000000000000000000009B2);
+		Counter(0x5615dEB798BB3E4dFa0139dFa1b3D433Cc23b72f).setNumber(996881781832960761274744263729582347);
+ invariant_increment() (runs: 0, calls: 0, reverts: 0)
+
+Encountered a total of 1 failing tests, 0 tests succeeded
+
+"#]],
+    );
+
+    // Persisted failures should be able to switch output.
+    prj.update_config(|config| {
+        config.invariant.show_solidity = false;
+    });
+    cmd.forge_fuse().args(["test", "--mt", "invariant_increment"]).assert_failure().stdout_eq(
+        str![[r#"
+...
+Failing tests:
+Encountered 1 failing test in test/InvariantSequenceLenTest.t.sol:InvariantSequenceLenTest
+[FAIL: invariant_increment replay failure]
+	[Sequence] (original: 4, shrunk: 4)
+		sender=0x00000000000000000000000000000000000018dE addr=[src/Counter.sol:Counter]0x5615dEB798BB3E4dFa0139dFa1b3D433Cc23b72f calldata=setNumber(uint256) args=[1931387396117645594923 [1.931e21]]
+		sender=0x00000000000000000000000000000000000009d5 addr=[src/Counter.sol:Counter]0x5615dEB798BB3E4dFa0139dFa1b3D433Cc23b72f calldata=increment() args=[]
+		sender=0x0000000000000000000000000000000000000105 addr=[src/Counter.sol:Counter]0x5615dEB798BB3E4dFa0139dFa1b3D433Cc23b72f calldata=increment() args=[]
+		sender=0x00000000000000000000000000000000000009B2 addr=[src/Counter.sol:Counter]0x5615dEB798BB3E4dFa0139dFa1b3D433Cc23b72f calldata=setNumber(uint256) args=[996881781832960761274744263729582347 [9.968e35]]
+ invariant_increment() (runs: 1, calls: 1, reverts: 1)
+
+Encountered a total of 1 failing tests, 0 tests succeeded
+
+"#]],
+    );
 });
diff --git a/crates/forge/tests/it/test_helpers.rs b/crates/forge/tests/it/test_helpers.rs
index 3488eca2fedf..0712ea73bd97 100644
--- a/crates/forge/tests/it/test_helpers.rs
+++ b/crates/forge/tests/it/test_helpers.rs
@@ -149,6 +149,7 @@ impl ForgeTestProfile {
             ),
             show_metrics: false,
             timeout: None,
+            show_solidity: false,
         };
 
         config.sanitized()