Repository: camshaft/bolero (c)

Issue:
new "random" engine
Bolero's standard engines, like AFL and libfuzzer, use coverage-guided fuzzing under the hood for input generation.

I'd like a new engine that has the same interface as these others, but just uses normal randomness under the hood for input generation. I realize that `cargo test` works this way now, but it is limited to running 1000 tests, not more, and it also runs saved corpus tests, which I don't necessarily want.

Pull request description:
feat: add random engine
resolves #179

Patch:
diff --git a/bin/cargo-bolero/src/engine.rs b/bin/cargo-bolero/src/engine.rs
index c841614..0618e11 100644
--- a/bin/cargo-bolero/src/engine.rs
+++ b/bin/cargo-bolero/src/engine.rs
@@ -14,6 +14,8 @@ pub enum Engine {
 
     #[cfg(feature = "kani")]
     Kani,
+
+    Random,
 }
 
 impl Engine {
@@ -29,6 +31,8 @@ impl Engine {
 
             #[cfg(feature = "kani")]
             Self::Kani => crate::kani::test(selection, args),
+
+            Self::Random => crate::random::test(selection, args),
         }
     }
 
@@ -44,10 +48,27 @@ impl Engine {
 
             #[cfg(feature = "kani")]
             Self::Kani => Ok(()),
+
+            Self::Random => Ok(()),
         }
     }
 }
 
+macro_rules! optional_engine {
+    ($lower:literal, $upper:ident) => {{
+        #[cfg(feature = $lower)]
+        let v = Ok(Self::$upper);
+
+        #[cfg(not(feature = $lower))]
+        let v = Err(format!(
+            "cargo-bolero was not built with `{}` feature",
+            $lower
+        ));
+
+        v
+    }};
+}
+
 impl FromStr for Engine {
     type Err = String;
 
@@ -55,16 +76,21 @@ impl FromStr for Engine {
         match value {
             "libfuzzer" => Ok(Self::Libfuzzer),
 
-            #[cfg(feature = "afl")]
-            "afl" => Ok(Self::Afl),
+            "afl" => {
+                optional_engine!("afl", Afl)
+            }
 
-            #[cfg(feature = "honggfuzz")]
-            "honggfuzz" => Ok(Self::Honggfuzz),
+            "honggfuzz" => {
+                optional_engine!("honggfuzz", Honggfuzz)
+            }
 
-            #[cfg(feature = "kani")]
-            "kani" => Ok(Self::Kani),
+            "kani" => {
+                optional_engine!("kani", Kani)
+            }
+
+            "random" => Ok(Self::Random),
 
-            _ => Err(format!("invalid fuzzer {:?}", value)),
+            _ => Err(format!("invalid engine {:?}", value)),
         }
     }
 }
diff --git a/bin/cargo-bolero/src/main.rs b/bin/cargo-bolero/src/main.rs
index 8362334..1fb6097 100644
--- a/bin/cargo-bolero/src/main.rs
+++ b/bin/cargo-bolero/src/main.rs
@@ -15,6 +15,7 @@ mod list;
 mod manifest;
 mod new;
 mod project;
+mod random;
 mod reduce;
 mod selection;
 mod test;
diff --git a/bin/cargo-bolero/src/random.rs b/bin/cargo-bolero/src/random.rs
new file mode 100644
index 0000000..37f2003
--- /dev/null
+++ b/bin/cargo-bolero/src/random.rs
@@ -0,0 +1,40 @@
+use crate::{exec, test, Selection};
+use anyhow::Result;
+
+const FLAGS: &[&str] = &["--cfg fuzzing_random"];
+
+pub(crate) fn test(selection: &Selection, test_args: &test::Args) -> Result<()> {
+    let test_target = selection.test_target(FLAGS, "random")?;
+
+    let mut cmd = test_target.command();
+
+    macro_rules! optional_arg {
+        ($arg:ident, $env:expr) => {
+            if let Some(v) = test_args.$arg {
+                cmd.env($env, v.to_string());
+            }
+        };
+    }
+
+    optional_arg!(seed, "BOLERO_RANDOM_SEED");
+    optional_arg!(runs, "BOLERO_RANDOM_ITERATIONS");
+    optional_arg!(max_input_length, "BOLERO_RANDOM_MAX_LEN");
+
+    // TODO implement other options
+    /*
+
+    /// Run the engine for a specified duration. If unspecified
+    /// it will continue until manually stopped.
+    #[structopt(short = "T")]
+    pub time: Option<Duration>,
+
+    /// Maximum amount of time to run a test target before
+    /// failing
+    #[structopt(short, long, default_value = "10s")]
+    pub timeout: Duration,
+    */
+
+    exec(cmd)?;
+
+    Ok(())
+}
diff --git a/lib/bolero-engine/src/rng.rs b/lib/bolero-engine/src/rng.rs
index e024dec..10f8c51 100644
--- a/lib/bolero-engine/src/rng.rs
+++ b/lib/bolero-engine/src/rng.rs
@@ -2,6 +2,39 @@ use crate::{driver, panic, ByteSliceTestInput, Engine, TargetLocation, Test};
 use core::fmt::Debug;
 use rand::{rngs::StdRng, Rng, RngCore, SeedableRng};
 
+#[derive(Clone, Copy)]
+pub struct Options {
+    pub iterations: Option<usize>,
+    pub max_len: Option<usize>,
+    pub seed: Option<u64>,
+}
+
+impl Default for Options {
+    fn default() -> Self {
+        Self {
+            iterations: get_var("BOLERO_RANDOM_ITERATIONS"),
+            max_len: get_var("BOLERO_RANDOM_MAX_LEN"),
+            seed: get_var("BOLERO_RANDOM_SEED"),
+        }
+    }
+}
+
+impl Options {
+    pub fn iterations_or_default(&self) -> usize {
+        // RNG tests are really slow with miri so we limit the number of iterations
+        self.iterations
+            .unwrap_or(if cfg!(miri) { 25 } else { 1000 })
+    }
+
+    pub fn max_len_or_default(&self) -> usize {
+        self.max_len.unwrap_or(if cfg!(miri) { 1024 } else { 4096 })
+    }
+
+    pub fn seed_or_rand(&self) -> u64 {
+        self.seed.unwrap_or_else(|| rand::thread_rng().next_u64())
+    }
+}
+
 /// Test engine implementation using a RNG.
 ///
 /// The inputs will only be derived from the `seed` field.
@@ -16,28 +49,16 @@ pub struct RngEngine {
 
 impl Default for RngEngine {
     fn default() -> Self {
-        if cfg!(miri) {
-            // RNG tests are really slow with miri so we limit the number of iterations
-            let iterations = get_var("BOLERO_RANDOM_ITERATIONS").unwrap_or(25);
-            let max_len = get_var("BOLERO_RANDOM_MAX_LEN").unwrap_or(1024);
-            let seed =
-                get_var("BOLERO_RANDOM_SEED").unwrap_or_else(|| rand::thread_rng().next_u64());
-
-            return Self {
-                iterations,
-                max_len,
-                seed,
-            };
-        }
-
-        let iterations = get_var("BOLERO_RANDOM_ITERATIONS").unwrap_or(1000);
-        let max_len = get_var("BOLERO_RANDOM_MAX_LEN").unwrap_or(4096);
-        let seed = get_var("BOLERO_RANDOM_SEED").unwrap_or_else(|| rand::thread_rng().next_u64());
+        Options::default().into()
+    }
+}
 
+impl From<Options> for RngEngine {
+    fn from(options: Options) -> Self {
         Self {
-            iterations,
-            max_len,
-            seed,
+            iterations: options.iterations_or_default(),
+            max_len: options.max_len_or_default(),
+            seed: options.seed_or_rand(),
         }
     }
 }
diff --git a/lib/bolero/Cargo.toml b/lib/bolero/Cargo.toml
index 083a404..9a8fd20 100644
--- a/lib/bolero/Cargo.toml
+++ b/lib/bolero/Cargo.toml
@@ -31,6 +31,10 @@ bolero-libfuzzer = { version = "0.9", path = "../bolero-libfuzzer" }
 [target.'cfg(fuzzing_honggfuzz)'.dependencies]
 bolero-honggfuzz = { version = "0.9", path = "../bolero-honggfuzz" }
 
+[target.'cfg(fuzzing_random)'.dependencies]
+bolero-engine = { version = "0.9", path = "../bolero-engine", features = ["rng"] }
+rand = { version = "^0.8" }
+
 [target.'cfg(kani)'.dependencies]
 bolero-kani = { version = "0.9", path = "../bolero-kani" }
 


Test patch:
diff --git a/bin/cargo-bolero/src/test.rs b/bin/cargo-bolero/src/test.rs
index 6087ba2..60f347e 100644
--- a/bin/cargo-bolero/src/test.rs
+++ b/bin/cargo-bolero/src/test.rs
@@ -21,7 +21,7 @@ pub struct Test {
 #[derive(Debug, StructOpt)]
 pub struct Args {
     /// Run the engine with an initial seed
-    #[structopt(short = "S")]
+    #[structopt(short = "S", env = "BOLERO_RANDOM_SEED")]
     pub seed: Option<usize>,
 
     /// Run the engine for a specified number of runs. If unspecified
diff --git a/lib/bolero/src/test/input.rs b/lib/bolero/src/test/input.rs
index d9d1eed..0f89008 100644
--- a/lib/bolero/src/test/input.rs
+++ b/lib/bolero/src/test/input.rs
@@ -1,3 +1,5 @@
+#![cfg_attr(fuzzing_random, allow(dead_code))]
+
 use bolero_engine::RngInput;
 use bolero_generator::{driver, TypeGenerator};
 use rand::{rngs::StdRng, SeedableRng};
diff --git a/lib/bolero/src/test/mod.rs b/lib/bolero/src/test/mod.rs
index 54c3a98..360135c 100644
--- a/lib/bolero/src/test/mod.rs
+++ b/lib/bolero/src/test/mod.rs
@@ -1,13 +1,15 @@
-use bolero_engine::{
-    driver, rng::RngEngine, test_failure::TestFailure, ByteSliceTestInput, Engine, Never,
-    TargetLocation, Test,
-};
+#![cfg_attr(fuzzing_random, allow(dead_code))]
+
+use bolero_engine::{driver, rng, test_failure::TestFailure, Engine, TargetLocation, Test};
 use core::iter::empty;
 use std::path::PathBuf;
 
 mod input;
 use input::*;
 
+#[cfg(any(fuzzing_random, test))]
+mod report;
+
 /// Engine implementation which mimics Rust's default test
 /// harness. By default, the test inputs will include any present
 /// `corpus` and `crashes` files, as well as generating
@@ -57,56 +59,106 @@ impl TestEngine {
             })
     }
 
-    fn rng_tests(&self) -> impl Iterator<Item = NamedTest> {
+    fn rng_tests(&self, config: rng::Options) -> impl Iterator<Item = RngTest> {
         use rand::{rngs::StdRng, RngCore, SeedableRng};
 
-        let rng_info = RngEngine::default();
-        let mut seed_rng = StdRng::seed_from_u64(rng_info.seed);
+        let iterations = config.iterations_or_default();
+        let max_len = config.max_len_or_default();
+        let seed = config.seed_or_rand();
+        let mut seed_rng = StdRng::seed_from_u64(seed);
 
-        (0..rng_info.iterations)
-            .scan(rng_info.seed, move |state, _index| {
+        (0..iterations)
+            .scan(seed, move |state, _index| {
                 let seed = *state;
                 *state = seed_rng.next_u64();
                 Some(seed)
             })
-            .map(move |seed| NamedTest {
-                name: format!("[BOLERO_RANDOM_SEED={}]", seed),
-                data: TestInput::RngTest(RngTest {
-                    seed,
-                    max_len: rng_info.max_len,
-                }),
-            })
+            .map(move |seed| input::RngTest { seed, max_len })
     }
 
     fn tests(&self) -> Vec<NamedTest> {
+        let rng_tests = self
+            .rng_tests(Default::default())
+            .map(move |test| NamedTest {
+                name: format!("[BOLERO_RANDOM_SEED={}]", test.seed),
+                data: TestInput::RngTest(test),
+            });
+
         empty()
             .chain(self.file_tests(["crashes"].iter().cloned()))
             .chain(self.file_tests(["afl_state", "crashes"].iter().cloned()))
             .chain(self.file_tests(["afl_state", "hangs"].iter().cloned()))
             .chain(self.file_tests(["corpus"].iter().cloned()))
             .chain(self.file_tests(["afl_state", "queue"].iter().cloned()))
-            .chain(self.rng_tests())
+            .chain(rng_tests)
             .collect()
     }
-}
 
-fn progress() {
-    if cfg!(miri) {
-        use std::io::{stderr, Write};
+    #[cfg(any(fuzzing_random, test))]
+    #[cfg_attr(not(fuzzing_random), allow(dead_code))]
+    fn run_fuzzer<T>(self, mut test: T, options: driver::Options) -> bolero_engine::Never
+    where
+        T: Test,
+        T::Value: core::fmt::Debug,
+    {
+        bolero_engine::panic::set_hook();
+        bolero_engine::panic::forward_panic(false);
 
-        // miri doesn't capture explicit writes to stderr
-        #[allow(clippy::explicit_write)]
-        let _ = write!(stderr(), ".");
-    }
-}
+        let options = &options;
 
-impl<T: Test> Engine<T> for TestEngine
-where
-    T::Value: core::fmt::Debug,
-{
-    type Output = Never;
+        let mut buffer = vec![];
+        let mut testfn = |conf: &input::RngTest| {
+            let mut input = conf.input(&mut buffer, options);
+            test.test(&mut input).map_err(|error| {
+                let seed = Some(conf.seed);
+                // reseed the input and buffer the rng for shrinking
+                let mut input = conf.buffered_input(&mut buffer, options);
+                let _ = test.generate_value(&mut input);
+
+                let shrunken = test.shrink(buffer.clone(), seed, options);
+
+                if let Some(shrunken) = shrunken {
+                    format!("{:#}", shrunken)
+                } else {
+                    buffer.clear();
+                    let mut input = conf.input(&mut buffer, options);
+                    let input = test.generate_value(&mut input);
+                    format!("{:#}", TestFailure { seed, error, input })
+                }
+            })
+        };
+
+        let mut report = report::Report::default();
+        report.spawn_timer();
 
-    fn run(self, mut test: T, options: driver::Options) -> Self::Output {
+        let inputs = {
+            let mut config = rng::Options::default();
+            if config.iterations.is_none() {
+                config.iterations = Some(usize::MAX);
+            }
+            self.rng_tests(config)
+        };
+
+        for input in inputs {
+            match testfn(&input) {
+                Ok(is_valid) => {
+                    report.on_result(is_valid);
+                }
+                Err(err) => {
+                    bolero_engine::panic::forward_panic(true);
+                    eprintln!("{}", err);
+                    panic!("test failed");
+                }
+            }
+        }
+    }
+
+    #[cfg(not(fuzzing_random))]
+    fn run_tests<T>(self, mut test: T, options: driver::Options)
+    where
+        T: Test,
+        T::Value: core::fmt::Debug,
+    {
         let mut file_options = options.clone();
         let mut rng_options = options.clone();
 
@@ -131,7 +183,7 @@ where
                 TestInput::FileTest(file) => {
                     file.read_into(&mut buffer);
 
-                    let mut input = ByteSliceTestInput::new(&buffer, file_options);
+                    let mut input = bolero_engine::ByteSliceTestInput::new(&buffer, file_options);
                     test.test(&mut input).map_err(|error| {
                         let shrunken = test.shrink(buffer.clone(), data.seed(), file_options);
 
@@ -192,5 +244,36 @@ where
                 panic!("test failed: {:?}", test.name);
             }
         }
+
+        fn progress() {
+            if cfg!(miri) {
+                use std::io::{stderr, Write};
+
+                // miri doesn't capture explicit writes to stderr
+                #[allow(clippy::explicit_write)]
+                let _ = write!(stderr(), ".");
+            }
+        }
+    }
+}
+
+impl<T> Engine<T> for TestEngine
+where
+    T: Test,
+    T::Value: core::fmt::Debug,
+{
+    #[cfg(fuzzing_random)]
+    type Output = bolero_engine::Never;
+    #[cfg(not(fuzzing_random))]
+    type Output = ();
+
+    #[cfg(fuzzing_random)]
+    fn run(self, test: T, options: driver::Options) -> Self::Output {
+        self.run_fuzzer(test, options)
+    }
+
+    #[cfg(not(fuzzing_random))]
+    fn run(self, test: T, options: driver::Options) -> Self::Output {
+        self.run_tests(test, options)
     }
 }
diff --git a/lib/bolero/src/test/report.rs b/lib/bolero/src/test/report.rs
new file mode 100644
index 0000000..10b4d88
--- /dev/null
+++ b/lib/bolero/src/test/report.rs
@@ -0,0 +1,69 @@
+#![cfg_attr(not(fuzzing_random), allow(dead_code))]
+
+use std::{
+    sync::{
+        atomic::{AtomicBool, Ordering},
+        Arc,
+    },
+    time::Duration,
+};
+
+#[derive(Default)]
+pub struct Report {
+    total_runs: u64,
+    window_runs: u64,
+    total_valid: u64,
+    window_valid: u64,
+    should_print: Arc<AtomicBool>,
+}
+
+impl Report {
+    pub fn spawn_timer(&self) {
+        let should_print = self.should_print.clone();
+        std::thread::spawn(move || {
+            while Arc::strong_count(&should_print) > 1 {
+                std::thread::sleep(Duration::from_secs(1));
+                should_print.store(true, Ordering::Relaxed);
+            }
+        });
+    }
+
+    pub fn on_result(&mut self, is_valid: bool) {
+        self.window_runs += 1;
+        if is_valid {
+            self.window_valid += 1;
+        }
+
+        // check the should_print every 1024 runs
+        if self.window_runs % 1024 != 0 {
+            return;
+        }
+
+        if !self.should_print.swap(false, Ordering::Relaxed) {
+            return;
+        }
+
+        self.total_runs += self.window_runs;
+        self.total_valid += self.window_valid;
+
+        // only report valid percentage if we drop below 100%
+        if self.total_runs == self.total_valid {
+            println!("#{}\titerations/s: {}", self.total_runs, self.window_runs);
+        } else {
+            let total_perc = self.total_valid as f32 / self.total_runs as f32 * 100.0;
+            let window_perc = self.window_valid as f32 / self.window_runs as f32 * 100.0;
+            println!(
+                "#{}\titerations/s: {} valid: {} ({:.2}%) valid/s: {} ({:.2}%)",
+                self.total_runs,
+                self.window_runs,
+                self.total_valid,
+                total_perc,
+                self.window_valid,
+                window_perc,
+            );
+        }
+
+        self.window_runs = 0;
+        self.window_valid = 0;
+    }
+}
diff --git a/tests/src/engines.rs b/tests/src/engines.rs
index 5a24049..da174c8 100644
--- a/tests/src/engines.rs
+++ b/tests/src/engines.rs
@@ -4,7 +4,7 @@ use xshell::{cmd, Shell};
 pub fn test() -> Result {
     let is_nightly = env::rustc_build().map_or(false, |b| b == "nightly");
 
-    for engine in ["libfuzzer", "afl", "honggfuzz", "kani"] {
+    for engine in ["random", "libfuzzer", "afl", "honggfuzz", "kani"] {
         // TODO fix honggfuzz
         if engine == "honggfuzz" {
             continue;