Repository: wasmerio/wasmer (language: rust)

Issue:
Feat: Add compilation times benchmarks
We must benchmark the compilation times with three main backends (`singlepass`, `cranelift` and `llvm`) for a selection of packages. 


Discussion / hints:

Fix patch:
diff --git a/Cargo.toml b/Cargo.toml
index c16b5b27e3f..4480d9fbb97 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -154,7 +154,7 @@ tracing-subscriber = { version = "0.3", default-features = false, features = [
 	"env-filter",
 	"fmt",
 ] }
-reqwest.workspace = true
+reqwest = {workspace = true, features = ["blocking", "rustls-tls"]}
 
 [features]
 # Don't add the compiler features in default, please add them on the Makefile
@@ -220,7 +220,11 @@ opt-level = 3
 opt-level = 3
 
 [[bench]]
-name = "compiler_run"
+name = "run"
+harness = false
+
+[[bench]]
+name = "compile"
 harness = false
 
 [[bench]]
diff --git a/benches/compile.rs b/benches/compile.rs
new file mode 100644
index 00000000000..4bcafbd0024
--- /dev/null
+++ b/benches/compile.rs
@@ -0,0 +1,149 @@
+use criterion::{
+    criterion_group, criterion_main, measurement::WallTime, BenchmarkGroup, Criterion,
+};
+use wasmer::*;
+
+static BENCHMARKS_ARTIFACTS_BASE_URL: &str = "https://pub-083d1a0568d446d1aa5b2e07bd16983b.r2.dev";
+
+fn get_engine() -> Engine {
+    #[cfg(feature = "llvm")]
+    return LLVM::new().into();
+
+    #[cfg(feature = "singlepass")]
+    return Singlepass::new().into();
+
+    #[cfg(feature = "cranelift")]
+    return Cranelift::new().into();
+
+    #[cfg(not(any(feature = "cranelift", feature = "llvm", feature = "singlepass")))]
+    return Default::default();
+}
+
+pub fn compile_wasm(c: &mut BenchmarkGroup<'_, WallTime>, module: &[u8], name: &str) {
+    c.bench_function(name, |b| {
+        let engine = get_engine();
+
+        b.iter(|| {
+            let store = Store::new(engine.clone());
+            assert!(Module::new(&store, module).is_ok());
+        })
+    });
+}
+
+pub fn download_and_compile_small(c: &mut BenchmarkGroup<'_, WallTime>) {
+    let name = if cfg!(feature = "cranelift") {
+        "cranelift"
+    } else if cfg!(feature = "llvm") {
+        "llvm"
+    } else if cfg!(feature = "singlepass") {
+        "singlepass"
+    } else if cfg!(feature = "v8") {
+        "v8"
+    } else if cfg!(feature = "wamr") {
+        "wamr"
+    } else if cfg!(feature = "wasmi") {
+        "wasmi"
+    } else {
+        panic!("Unrecognized backend!")
+    };
+
+    let modules = [
+        "counter", "primes", "fib_rec", "fib_iter", "bulk_ops", "matmul",
+    ];
+
+    for module in modules {
+        let bytes =
+            reqwest::blocking::get(format!("{BENCHMARKS_ARTIFACTS_BASE_URL}/{module}.wasm"))
+                .unwrap()
+                .bytes()
+                .unwrap();
+        compile_wasm(c, bytes.as_ref(), &format!("exec/{name}/{module}"));
+    }
+}
+
+pub fn download_and_compile_medium(c: &mut BenchmarkGroup<'_, WallTime>) {
+    let name = if cfg!(feature = "cranelift") {
+        "cranelift"
+    } else if cfg!(feature = "llvm") {
+        "llvm"
+    } else if cfg!(feature = "singlepass") {
+        "singlepass"
+    } else if cfg!(feature = "v8") {
+        "v8"
+    } else if cfg!(feature = "wamr") {
+        "wamr"
+    } else if cfg!(feature = "wasmi") {
+        "wasmi"
+    } else {
+        panic!("Unrecognized backend!")
+    };
+
+    let modules = ["bash", "irb", "argon2"];
+
+    for module in modules {
+        let bytes =
+            reqwest::blocking::get(format!("{BENCHMARKS_ARTIFACTS_BASE_URL}/{module}.wasm"))
+                .unwrap()
+                .bytes()
+                .unwrap();
+        compile_wasm(c, bytes.as_ref(), &format!("exec/{name}/{module}"));
+    }
+}
+
+pub fn download_and_compile_large(c: &mut BenchmarkGroup<'_, WallTime>) {
+    let name = if cfg!(feature = "cranelift") {
+        "cranelift"
+    } else if cfg!(feature = "llvm") {
+        "llvm"
+    } else if cfg!(feature = "singlepass") {
+        "singlepass"
+    } else if cfg!(feature = "v8") {
+        "v8"
+    } else if cfg!(feature = "wamr") {
+        "wamr"
+    } else if cfg!(feature = "wasmi") {
+        "wasmi"
+    } else {
+        panic!("Unrecognized backend!")
+    };
+
+    let modules = [
+        "winterjs",
+        "wasix_axum",
+        "static_web_server",
+        "s3_server",
+        "python",
+        "php",
+    ];
+
+    for module in modules {
+        let bytes =
+            reqwest::blocking::get(format!("{BENCHMARKS_ARTIFACTS_BASE_URL}/{module}.wasm"))
+                .unwrap()
+                .bytes()
+                .unwrap();
+        compile_wasm(c, bytes.as_ref(), &format!("exec/{name}/{module}"));
+    }
+}
+
+pub fn download_and_compile(c: &mut Criterion) {
+    {
+        let mut group = c.benchmark_group("large_benches");
+        group.sample_size(10);
+        download_and_compile_large(&mut group);
+    }
+
+    {
+        let mut group = c.benchmark_group("medium_benches");
+        group.sample_size(40);
+        download_and_compile_medium(&mut group);
+    }
+    {
+        let mut group = c.benchmark_group("small_benches");
+        group.sample_size(60);
+        download_and_compile_small(&mut group);
+    }
+}
+
+criterion_group!(benches, download_and_compile);
+criterion_main!(benches);
diff --git a/benches/deserialize_modules.rs b/benches/deserialize_modules.rs
index 1c45e87527d..10207a77237 100644
--- a/benches/deserialize_modules.rs
+++ b/benches/deserialize_modules.rs
@@ -4,61 +4,24 @@ use wasmer::*;
 
 static BENCHMARKS_ARTIFACTS_BASE_URL: &str = "https://pub-53a226d993e144159d6f8b993fe0cbf3.r2.dev";
 
-pub fn serialize_deserialize(c: &mut Criterion, small: &[u8], medium: &[u8], large: &[u8]) {
+pub fn serialize_deserialize(c: &mut Criterion, bytes: &[u8], name: &str) {
     let engine = Engine::default();
 
-    c.bench_function(&format!("rkyv_from-binary_small"), |b| {
+    c.bench_function(&format!("rkyv_from-binary_{name}"), |b| {
         b.iter(|| {
-            let module = Module::from_binary(&engine, small);
-            assert!(module.is_ok());
-            drop(module);
-        })
-    });
-
-    c.bench_function(&format!("rkyv_from-binary_medium"), |b| {
-        b.iter(|| {
-            let module = Module::from_binary(&engine, medium);
-            assert!(module.is_ok());
-            drop(module);
-        })
-    });
-
-    c.bench_function(&format!("rkyv_from-binary_large"), |b| {
-        b.iter(|| {
-            let module = Module::from_binary(&engine, large);
+            let module = Module::from_binary(&engine, bytes);
             assert!(module.is_ok());
             drop(module);
         })
     });
 }
 
-pub fn serialize_deserialize_unchecked(
-    c: &mut Criterion,
-    small: &[u8],
-    medium: &[u8],
-    large: &[u8],
-) {
+pub fn serialize_deserialize_unchecked(c: &mut Criterion, bytes: &[u8], name: &str) {
     let engine = Engine::default();
 
-    c.bench_function(&format!("rkyv_from-binary-unchecked_small"), |b| {
+    c.bench_function(&format!("rkyv_from-binary-unchecked_{name}"), |b| {
         b.iter(|| unsafe {
-            let module = Module::from_binary_unchecked(&engine, small);
-            assert!(module.is_ok());
-            drop(module);
-        })
-    });
-
-    c.bench_function(&format!("rkyv_from-binary-unchecked_medium"), |b| {
-        b.iter(|| unsafe {
-            let module = Module::from_binary_unchecked(&engine, medium);
-            assert!(module.is_ok());
-            drop(module);
-        })
-    });
-
-    c.bench_function(&format!("rkyv_from-binary-unchecked_large"), |b| {
-        b.iter(|| unsafe {
-            let module = Module::from_binary_unchecked(&engine, large);
+            let module = Module::from_binary_unchecked(&engine, bytes);
             assert!(module.is_ok());
             drop(module);
         })
@@ -66,26 +29,17 @@ pub fn serialize_deserialize_unchecked(
 }
 
 pub fn download_and_run(c: &mut Criterion) {
-    let small = reqwest::blocking::get(BENCHMARKS_ARTIFACTS_BASE_URL.to_owned() + "/small.wasm")
-        .unwrap()
-        .bytes()
-        .unwrap();
-
-    let medium = reqwest::blocking::get(BENCHMARKS_ARTIFACTS_BASE_URL.to_owned() + "/medium.wasm")
-        .unwrap()
-        .bytes()
-        .unwrap();
-
-    let large = reqwest::blocking::get(BENCHMARKS_ARTIFACTS_BASE_URL.to_owned() + "/large.wasm")
-        .unwrap()
-        .bytes()
-        .unwrap();
-
-    let small = small.as_ref();
-    let medium = medium.as_ref();
-    let large = large.as_ref();
-    serialize_deserialize(c, small, medium, large);
-    serialize_deserialize_unchecked(c, small, medium, large);
+    let modules = ["small", "medium", "large"];
+
+    for module in modules {
+        let bytes =
+            reqwest::blocking::get(format!("{BENCHMARKS_ARTIFACTS_BASE_URL}/{module}.wasm"))
+                .unwrap()
+                .bytes()
+                .unwrap();
+        serialize_deserialize(c, &bytes, module);
+        serialize_deserialize_unchecked(c, &bytes, module);
+    }
 }
 
 criterion_group!(
diff --git a/benches/compiler_run.rs b/benches/run.rs
similarity index 64%
rename from benches/compiler_run.rs
rename to benches/run.rs
index 52b6c13371f..c3fbe08cf08 100644
--- a/benches/compiler_run.rs
+++ b/benches/run.rs
@@ -1,7 +1,7 @@
 use criterion::{criterion_group, criterion_main, Criterion};
 use wasmer::*;
 
-static BENCHMARKS_ARTIFACTS_BASE_URL: &str = "https://pub-53a226d993e144159d6f8b993fe0cbf3.r2.dev";
+static BENCHMARKS_ARTIFACTS_BASE_URL: &str = "https://pub-083d1a0568d446d1aa5b2e07bd16983b.r2.dev";
 
 fn get_engine() -> Engine {
     #[cfg(feature = "llvm")]
@@ -52,20 +52,24 @@ pub fn download_and_run(c: &mut Criterion) {
         panic!("Unrecognized backend!")
     };
 
-    let bytes = include_bytes!("./mods/counter.wasm");
-    run_fn(c, bytes, &format!("exec/{name}/counter"), 5_000_000);
-    let bytes = include_bytes!("./mods/primes.wasm");
-    run_fn(c, bytes, &format!("exec/{name}/primes"), 1_000);
-    let bytes = include_bytes!("./mods/fib_rec.wasm");
-    run_fn(c, bytes, &format!("exec/{name}/fib_rec"), 40);
-    let bytes = include_bytes!("./mods/fib_iter.wasm");
-    run_fn(c, bytes, &format!("exec/{name}/fib_iter"), 2_000_000);
-    let bytes = include_bytes!("./mods/bulk_ops.wasm");
-    run_fn(c, bytes, &format!("exec/{name}/bulk_ops"), 5_000);
-    let bytes = include_bytes!("./mods/matmul.wasm");
-    run_fn(c, bytes, &format!("exec/{name}/matmul"), 200);
-    let bytes = include_bytes!("./mods/argon2.wasm");
-    run_fn(c, bytes, &format!("exec/{name}/argon2"), 1);
+    let modules = [
+        ("counter", 5_000_000),
+        ("primes", 1_000),
+        ("fib_rec", 40),
+        ("fib_iter", 2_000_000),
+        ("bulk_ops", 5_000),
+        ("matmul", 200),
+        ("argon2", 1),
+    ];
+
+    for (module, arg) in modules {
+        let bytes =
+            reqwest::blocking::get(format!("{BENCHMARKS_ARTIFACTS_BASE_URL}/{module}.wasm"))
+                .unwrap()
+                .bytes()
+                .unwrap();
+        run_fn(c, bytes.as_ref(), &format!("exec/{name}/{module}"), arg);
+    }
 }
 
 criterion_group!(


Test patch: