Persist and replay fuzz failure

Author: grandizzy

crates/config/src/fuzz.rs

@@ -26,8 +26,6 @@ pub struct FuzzConfig {
     pub gas_report_samples: u32,
     /// Path where fuzz failures are recorded and replayed.
     pub failure_persist_dir: Option<PathBuf>,
-    /// Name of the file to record fuzz failures, defaults to `failures`.
-    pub failure_persist_file: Option<String>,
     /// show `console.log` in fuzz test, defaults to `false`
     pub show_logs: bool,
     /// Optional timeout (in seconds) for each property test
@@ -44,7 +42,6 @@ impl Default for FuzzConfig {
             dictionary: FuzzDictionaryConfig::default(),
             gas_report_samples: 256,
             failure_persist_dir: None,
-            failure_persist_file: None,
             show_logs: false,
             timeout: None,
         }
@@ -54,11 +51,7 @@ impl Default for FuzzConfig {
 impl FuzzConfig {
     /// Creates fuzz configuration to write failures in `{PROJECT_ROOT}/cache/fuzz` dir.
     pub fn new(cache_dir: PathBuf) -> Self {
-        Self {
-            failure_persist_dir: Some(cache_dir),
-            failure_persist_file: Some("failures".to_string()),
-            ..Default::default()
-        }
+        Self { failure_persist_dir: Some(cache_dir), ..Default::default() }
     }
 }
 

crates/evm/evm/src/executors/fuzz/mod.rs

@@ -54,14 +54,16 @@ pub struct FuzzTestData {
 /// inputs, until it finds a counterexample. The provided [`TestRunner`] contains all the
 /// configuration which can be overridden via [environment variables](proptest::test_runner::Config)
 pub struct FuzzedExecutor {
-    /// The EVM executor
+    /// The EVM executor.
     executor: Executor,
     /// The fuzzer
     runner: TestRunner,
-    /// The account that calls tests
+    /// The account that calls tests.
     sender: Address,
-    /// The fuzz configuration
+    /// The fuzz configuration.
     config: FuzzConfig,
+    /// The persisted counterexample to be replayed, if any.
+    persisted_failure: Option<BaseCounterExample>,
 }
 
 impl FuzzedExecutor {
@@ -71,15 +73,16 @@ impl FuzzedExecutor {
         runner: TestRunner,
         sender: Address,
         config: FuzzConfig,
+        persisted_failure: Option<BaseCounterExample>,
     ) -> Self {
-        Self { executor, runner, sender, config }
+        Self { executor, runner, sender, config, persisted_failure }
     }
 
     /// Fuzzes the provided function, assuming it is available at the contract at `address`
     /// If `should_fail` is set to `true`, then it will stop only when there's a success
     /// test case.
     ///
-    /// Returns a list of all the consumed gas and calldata of every fuzz case
+    /// Returns a list of all the consumed gas and calldata of every fuzz case.
     pub fn fuzz(
         &mut self,
         func: &Function,
@@ -116,57 +119,64 @@ impl FuzzedExecutor {
         let mut runs = 0;
         let mut rejects = 0;
         let mut run_failure = None;
+
         'stop: while continue_campaign(runs) {
-            let Ok(strategy) = strategy.new_tree(&mut self.runner) else {
-                run_failure = Some(TestCaseError::fail("no input generated to call fuzzed target"));
-                break 'stop;
-            };
+            // If counterexample recorded, replay it first, without incrementing runs.
+            let input = if let Some(failure) = self.persisted_failure.take() {
+                failure.calldata
+            } else {
+                // If running with progress, then increment current run.
+                if let Some(progress) = progress {
+                    progress.inc(1);
+                };
 
-            // If running with progress then increment current run.
-            if let Some(progress) = progress {
-                progress.inc(1);
-            };
+                runs += 1;
 
-            match self.single_fuzz(address, strategy.current()) {
-                Ok(fuzz_outcome) => {
-                    match fuzz_outcome {
-                        FuzzOutcome::Case(case) => {
-                            let mut data = execution_data.borrow_mut();
-                            data.gas_by_case.push((case.case.gas, case.case.stipend));
+                let Ok(strategy) = strategy.new_tree(&mut self.runner) else {
+                    run_failure =
+                        Some(TestCaseError::fail("no input generated to call fuzzed target"));
+                    break 'stop;
+                };
+                strategy.current()
+            };
 
-                            if data.first_case.is_none() {
-                                data.first_case.replace(case.case);
-                            }
+            match self.single_fuzz(address, input) {
+                Ok(fuzz_outcome) => match fuzz_outcome {
+                    FuzzOutcome::Case(case) => {
+                        let mut data = execution_data.borrow_mut();
+                        data.gas_by_case.push((case.case.gas, case.case.stipend));
 
-                            if let Some(call_traces) = case.traces {
-                                if data.traces.len() == max_traces_to_collect {
-                                    data.traces.pop();
-                                }
-                                data.traces.push(call_traces);
-                                data.breakpoints.replace(case.breakpoints);
-                            }
+                        if data.first_case.is_none() {
+                            data.first_case.replace(case.case);
+                        }
 
-                            if show_logs {
-                                data.logs.extend(case.logs);
+                        if let Some(call_traces) = case.traces {
+                            if data.traces.len() == max_traces_to_collect {
+                                data.traces.pop();
                             }
-
-                            HitMaps::merge_opt(&mut data.coverage, case.coverage);
-                            data.deprecated_cheatcodes = case.deprecated_cheatcodes;
+                            data.traces.push(call_traces);
+                            data.breakpoints.replace(case.breakpoints);
                         }
-                        FuzzOutcome::CounterExample(CounterExampleOutcome {
-                            exit_reason: status,
-                            counterexample: outcome,
-                            ..
-                        }) => {
-                            let reason = rd.maybe_decode(&outcome.1.result, status);
-                            execution_data.borrow_mut().logs.extend(outcome.1.logs.clone());
-                            execution_data.borrow_mut().counterexample = outcome;
-                            // HACK: we have to use an empty string here to denote `None`.
-                            run_failure = Some(TestCaseError::fail(reason.unwrap_or_default()));
-                            break 'stop;
+
+                        if show_logs {
+                            data.logs.extend(case.logs);
                         }
+
+                        HitMaps::merge_opt(&mut data.coverage, case.coverage);
+                        data.deprecated_cheatcodes = case.deprecated_cheatcodes;
                     }
-                }
+                    FuzzOutcome::CounterExample(CounterExampleOutcome {
+                        exit_reason: status,
+                        counterexample: outcome,
+                        ..
+                    }) => {
+                        let reason = rd.maybe_decode(&outcome.1.result, status);
+                        execution_data.borrow_mut().logs.extend(outcome.1.logs.clone());
+                        execution_data.borrow_mut().counterexample = outcome;
+                        run_failure = Some(TestCaseError::fail(reason.unwrap_or_default()));
+                        break 'stop;
+                    }
+                },
                 Err(err) => {
                     match err {
                         TestCaseError::Fail(_) => {
@@ -188,8 +198,6 @@ impl FuzzedExecutor {
                     }
                 }
             }
-
-            runs += 1;
         }
 
         let fuzz_result = execution_data.into_inner();

crates/forge/src/runner.rs

@@ -13,7 +13,7 @@ use alloy_primitives::{Address, Bytes, U256, address, map::HashMap};
 use eyre::Result;
 use foundry_common::{TestFunctionExt, TestFunctionKind, contracts::ContractsByAddress};
 use foundry_compilers::utils::canonicalized;
-use foundry_config::{Config, InvariantConfig};
+use foundry_config::Config;
 use foundry_evm::{
     constants::CALLER,
     decode::RevertDecoder,
@@ -31,9 +31,7 @@ use foundry_evm::{
     traces::{TraceKind, TraceMode, load_contracts},
 };
 use itertools::Itertools;
-use proptest::test_runner::{
-    FailurePersistence, FileFailurePersistence, RngAlgorithm, TestError, TestRng, TestRunner,
-};
+use proptest::test_runner::{RngAlgorithm, TestError, TestRng, TestRunner};
 use rayon::prelude::*;
 use serde::{Deserialize, Serialize};
 use std::{
@@ -726,8 +724,8 @@ impl<'a> FunctionRunner<'a> {
             abi: &self.cr.contract.abi,
         };
 
-        let (failure_dir, failure_file) = invariant_failure_paths(
-            invariant_config,
+        let (failure_dir, failure_file) = test_failure_paths(
+            invariant_config.failure_persist_dir.clone().unwrap(),
             self.cr.name,
             &invariant_contract.invariant_function.name,
         );
@@ -929,9 +927,23 @@ impl<'a> FunctionRunner<'a> {
             fuzz_config.runs,
         );
 
+        let (failure_dir, failure_file) = test_failure_paths(
+            fuzz_config.failure_persist_dir.clone().unwrap(),
+            self.cr.name,
+            &func.name,
+        );
+
+        // Load persisted counterexample, if any.
+        let persisted_failure =
+            foundry_common::fs::read_json_file::<BaseCounterExample>(failure_file.as_path()).ok();
         // Run fuzz test.
-        let mut fuzzed_executor =
-            FuzzedExecutor::new(self.executor.into_owned(), runner, self.tcfg.sender, fuzz_config);
+        let mut fuzzed_executor = FuzzedExecutor::new(
+            self.executor.into_owned(),
+            runner,
+            self.tcfg.sender,
+            fuzz_config,
+            persisted_failure,
+        );
         let result = fuzzed_executor.fuzz(
             func,
             &self.setup.fuzz_fixtures,
@@ -940,6 +952,18 @@ impl<'a> FunctionRunner<'a> {
             &self.cr.mcr.revert_decoder,
             progress.as_ref(),
         );
+
+        // Record counterexample.
+        if let Some(CounterExample::Single(counterexample)) = &result.counterexample {
+            if let Err(err) = foundry_common::fs::create_dir_all(failure_dir) {
+                error!(%err, "Failed to create fuzz failure dir");
+            } else if let Err(err) =
+                foundry_common::fs::write_json_file(failure_file.as_path(), counterexample)
+            {
+                error!(%err, "Failed to record call sequence");
+            }
+        }
+
         self.result.fuzz_result(result);
         self.result
     }
@@ -995,40 +1019,21 @@ impl<'a> FunctionRunner<'a> {
 
     fn fuzz_runner(&self) -> TestRunner {
         let config = &self.config.fuzz;
-        let failure_persist_path = config
-            .failure_persist_dir
-            .as_ref()
-            .unwrap()
-            .join(config.failure_persist_file.as_ref().unwrap())
-            .into_os_string()
-            .into_string()
-            .unwrap();
-        fuzzer_with_cases(
-            config.seed,
-            config.runs,
-            config.max_test_rejects,
-            Some(Box::new(FileFailurePersistence::Direct(failure_persist_path.leak()))),
-        )
+        fuzzer_with_cases(config.seed, config.runs, config.max_test_rejects)
     }
 
     fn invariant_runner(&self) -> TestRunner {
         let config = &self.config.invariant;
-        fuzzer_with_cases(self.config.fuzz.seed, config.runs, config.max_assume_rejects, None)
+        fuzzer_with_cases(self.config.fuzz.seed, config.runs, config.max_assume_rejects)
     }
 
     fn clone_executor(&self) -> Executor {
         self.executor.clone().into_owned()
     }
 }
 
-fn fuzzer_with_cases(
-    seed: Option<U256>,
-    cases: u32,
-    max_global_rejects: u32,
-    file_failure_persistence: Option<Box<dyn FailurePersistence>>,
-) -> TestRunner {
+fn fuzzer_with_cases(seed: Option<U256>, cases: u32, max_global_rejects: u32) -> TestRunner {
     let config = proptest::test_runner::Config {
-        failure_persistence: file_failure_persistence,
         cases,
         max_global_rejects,
         // Disable proptest shrink: for fuzz tests we provide single counterexample,
@@ -1077,18 +1082,13 @@ fn persisted_call_sequence(path: &Path, bytecode: &Bytes) -> Option<Vec<BaseCoun
     )
 }
 
-/// Helper functions to return canonicalized invariant failure paths.
-fn invariant_failure_paths(
-    config: &InvariantConfig,
+/// Helper functions to return canonicalized test failure paths.
+fn test_failure_paths(
+    persist_dir: PathBuf,
     contract_name: &str,
     invariant_name: &str,
 ) -> (PathBuf, PathBuf) {
-    let dir = config
-        .failure_persist_dir
-        .clone()
-        .unwrap()
-        .join("failures")
-        .join(contract_name.split(':').next_back().unwrap());
+    let dir = persist_dir.join("failures").join(contract_name.split(':').next_back().unwrap());
     let dir = canonicalized(dir);
     let file = canonicalized(dir.join(invariant_name));
     (dir, file)

crates/forge/tests/cli/config.rs

@@ -84,7 +84,6 @@ forgetest!(can_extract_config_values, |prj, cmd| {
             max_test_rejects: 100203,
             seed: Some(U256::from(1000)),
             failure_persist_dir: Some("test-cache/fuzz".into()),
-            failure_persist_file: Some("failures".to_string()),
             show_logs: false,
             ..Default::default()
         },
@@ -1099,7 +1098,6 @@ max_fuzz_dictionary_addresses = 15728640
 max_fuzz_dictionary_values = 6553600
 gas_report_samples = 256
 failure_persist_dir = "cache/fuzz"
-failure_persist_file = "failures"
 show_logs = false
 
 [invariant]
@@ -1211,7 +1209,6 @@ exclude = []
     "max_fuzz_dictionary_values": 6553600,
     "gas_report_samples": 256,
     "failure_persist_dir": "cache/fuzz",
-    "failure_persist_file": "failures",
     "show_logs": false,
     "timeout": null
   },

crates/forge/tests/it/fuzz.rs

@@ -150,9 +150,9 @@ async fn test_persist_fuzz_failure() {
         assert_eq!(initial_calldata, new_calldata, "run {i}");
     }
 
-    // write new failure in different file
+    // write new failure in different dir.
     let new_calldata = match run_fail!(|config| {
-        config.fuzz.failure_persist_file = Some("failure1".to_string());
+        config.fuzz.failure_persist_dir = Some(tempfile::tempdir().unwrap().keep())
     }) {
         Some(CounterExample::Single(counterexample)) => counterexample.calldata,
         _ => Bytes::new(),

crates/forge/tests/it/test_helpers.rs

@@ -127,7 +127,6 @@ impl ForgeTestProfile {
             },
             gas_report_samples: 256,
             failure_persist_dir: Some(tempfile::tempdir().unwrap().keep()),
-            failure_persist_file: Some("testfailure".to_string()),
             show_logs: false,
             timeout: None,
         };