Repository: tetratelabs/wazero
Pull request #164 (2022-01-26 01:55:58)

adds wasmer comparison in benchmarks
While both wasmer-go and wasmtime-go end up using wasm-tools for text
compilation, they do it slightly differently. This adds a dimension for
wasmer.

```
BenchmarkCodecExample/wat2wasm_via_text.DecodeModule->binary.EncodeModule
BenchmarkCodecExample/wat2wasm_via_text.DecodeModule->binary.EncodeModule-16         	   60103	     19967 ns/op	   12600 B/op	     456 allocs/op
BenchmarkCodecExample/wat2wasm_vs_wasmtime.Wat2Wasm
BenchmarkCodecExample/wat2wasm_vs_wasmtime.Wat2Wasm-16                               	   32911	     35606 ns/op	    1744 B/op	       3 allocs/op
BenchmarkCodecExample/wat2wasm_vs_wasmer.Wat2Wasm
BenchmarkCodecExample/wat2wasm_vs_wasmer.Wat2Wasm-16                                 	   31758	     37278 ns/op	    1776 B/op	       6 allocs/op
```

I also added this to the end-to-end benchmarks, adding an init test:
```
goos: darwin
goarch: amd64
pkg: github.com/tetratelabs/wazero/bench
cpu: Intel(R) Core(TM) i9-9880H CPU @ 2.30GHz
BenchmarkFacIter_Init
BenchmarkFacIter_Init/interpreter
BenchmarkFacIter_Init/interpreter-16         	   61927	     16852 ns/op
BenchmarkFacIter_Init/jit
BenchmarkFacIter_Init/jit-16                 	    4950	    221499 ns/op
BenchmarkFacIter_Init/wasmer-go
BenchmarkFacIter_Init/wasmer-go-16           	    3592	    323731 ns/op
BenchmarkFacIter_Init/wasmtime-go
BenchmarkFacIter_Init/wasmtime-go-16         	    3963	    296131 ns/op
BenchmarkFacIter_Invoke
BenchmarkFacIter_Invoke/interpreter
BenchmarkFacIter_Invoke/interpreter-16       	  622407	      1814 ns/op
BenchmarkFacIter_Invoke/jit
BenchmarkFacIter_Invoke/jit-16               	 5724092	       209.3 ns/op
BenchmarkFacIter_Invoke/wasmer-go
BenchmarkFacIter_Invoke/wasmer-go-16         	  961142	      1284 ns/op
BenchmarkFacIter_Invoke/wasmtime-go
BenchmarkFacIter_Invoke/wasmtime-go-16       	  550672	      2319 ns/op
```

codefromthecrypt: good point about the other bench. I will add wasmer to the factorial one also!
codefromthecrypt: I updated the description

Patch:
diff --git a/go.mod b/go.mod
index 30872227..a74fa96e 100644
--- a/go.mod
+++ b/go.mod
@@ -4,9 +4,11 @@ module github.com/tetratelabs/wazero
 go 1.16
 
 require (
-	// only used in wat benchmarks
+	// only used in benchmarks
 	github.com/bytecodealliance/wasmtime-go v0.32.0
 	github.com/stretchr/testify v1.7.0
 	// Once we reach some maturity, remove this dep and implement our own assembler.
 	github.com/twitchyliquid64/golang-asm v0.15.1
+	// only used in benchmarks
+	github.com/wasmerio/wasmer-go v1.0.4
 )
diff --git a/go.sum b/go.sum
index b2ae9bb6..04666e77 100644
--- a/go.sum
+++ b/go.sum
@@ -9,6 +9,8 @@ github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5Cc
 github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/twitchyliquid64/golang-asm v0.15.1 h1:SU5vSMR7hnwNxj24w34ZyCi/FmDZTkS4MhqMhdFk5YI=
 github.com/twitchyliquid64/golang-asm v0.15.1/go.mod h1:a1lVb/DtPvCB8fslRZhAngC2+aY1QWCk3Cedj/Gdt08=
+github.com/wasmerio/wasmer-go v1.0.4 h1:MnqHoOGfiQ8MMq2RF6wyCeebKOe84G88h5yv+vmxJgs=
+github.com/wasmerio/wasmer-go v1.0.4/go.mod h1:0gzVdSfg6pysA6QVp6iVRPTagC6Wq9pOE8J86WKb2Fk=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo=


Test patch:
diff --git a/bench/bench_fac_iter_test.go b/bench/bench_fac_iter_test.go
index b14ab2bf..2c916993 100644
--- a/bench/bench_fac_iter_test.go
+++ b/bench/bench_fac_iter_test.go
@@ -1,11 +1,13 @@
 package bench
 
 import (
-	"os"
+	_ "embed"
+	"errors"
 	"testing"
 
 	"github.com/bytecodealliance/wasmtime-go"
 	"github.com/stretchr/testify/require"
+	"github.com/wasmerio/wasmer-go/wasmer"
 
 	"github.com/tetratelabs/wazero/wasm"
 	binaryFormat "github.com/tetratelabs/wazero/wasm/binary"
@@ -13,33 +15,52 @@ import (
 	"github.com/tetratelabs/wazero/wasm/jit"
 )
 
+// facWasm is compiled from testdata/fac.wat
+//go:embed testdata/fac.wasm
+var facWasm []byte
+
 // TestFacIter ensures that the code in BenchmarkFacIter works as expected.
 func TestFacIter(t *testing.T) {
 	const in = 30
 	expValue := uint64(0x865df5dd54000000)
 	t.Run("iter", func(t *testing.T) {
-		store := newStoreForFacIterBench(jit.NewEngine())
+		store, err := newStoreForFacIterBench(jit.NewEngine())
+		require.NoError(t, err)
+
 		for i := 0; i < 10000; i++ {
 			res, _, err := store.CallFunction("test", "fac-iter", in)
-			if err != nil {
-				panic(err)
-			}
+			require.NoError(t, err)
 			require.Equal(t, expValue, res[0])
 		}
 	})
 
 	t.Run("jit", func(t *testing.T) {
-		store := newStoreForFacIterBench(jit.NewEngine())
+		store, err := newStoreForFacIterBench(jit.NewEngine())
+		require.NoError(t, err)
+
 		for i := 0; i < 10000; i++ {
 			res, _, err := store.CallFunction("test", "fac-iter", in)
-			if err != nil {
-				panic(err)
-			}
+			require.NoError(t, err)
 			require.Equal(t, expValue, res[0])
 		}
 	})
+
+	t.Run("wasmer-go", func(t *testing.T) {
+		store, instance, fn, err := newWasmerForFacIterBench()
+		require.NoError(t, err)
+		defer store.Close()
+		defer instance.Close()
+
+		for i := 0; i < 10000; i++ {
+			res, err := fn(in)
+			require.NoError(t, err)
+			require.Equal(t, int64(expValue), res)
+		}
+	})
+
 	t.Run("wasmtime-go", func(t *testing.T) {
-		store, run := newWasmtimeForFacIterBench()
+		store, run, err := newWasmtimeForFacIterBench()
+		require.NoError(t, err)
 		for i := 0; i < 10000; i++ {
 			res, err := run.Call(store, in)
 			if err != nil {
@@ -50,77 +71,151 @@ func TestFacIter(t *testing.T) {
 	})
 }
 
-// Benchmarks on the interative factorial calculation.
-func BenchmarkFacIter(b *testing.B) {
-	const in = 30
-	b.Run("wazeroir", func(b *testing.B) {
-		store := newStoreForFacIterBench(interpreter.NewEngine())
+// BenchmarkFacIter_Init tracks the time spent readying a function for use
+func BenchmarkFacIter_Init(b *testing.B) {
+	b.Run("interpreter", func(b *testing.B) {
 		b.ResetTimer()
 		for i := 0; i < b.N; i++ {
-			_, _, err := store.CallFunction("test", "fac-iter", in)
-			if err != nil {
-				panic(err)
+			if _, err := newStoreForFacIterBench(interpreter.NewEngine()); err != nil {
+				b.Fatal(err)
 			}
 		}
 	})
 	b.Run("jit", func(b *testing.B) {
-		store := newStoreForFacIterBench(jit.NewEngine())
 		b.ResetTimer()
 		for i := 0; i < b.N; i++ {
-			_, _, err := store.CallFunction("test", "fac-iter", in)
+			if _, err := newStoreForFacIterBench(jit.NewEngine()); err != nil {
+				b.Fatal(err)
+			}
+		}
+	})
+	b.Run("wasmer-go", func(b *testing.B) {
+		b.ResetTimer()
+		for i := 0; i < b.N; i++ {
+			store, instance, _, err := newWasmerForFacIterBench()
 			if err != nil {
-				panic(err)
+				b.Fatal(err)
 			}
+			store.Close()
+			instance.Close()
 		}
 	})
 	b.Run("wasmtime-go", func(b *testing.B) {
-		store, run := newWasmtimeForFacIterBench()
 		b.ResetTimer()
 		for i := 0; i < b.N; i++ {
-			_, err := run.Call(store, in)
-			if err != nil {
-				panic(err)
+			if _, _, err := newWasmtimeForFacIterBench(); err != nil {
+				b.Fatal(err)
 			}
 		}
 	})
 }
 
-func newStoreForFacIterBench(engine wasm.Engine) *wasm.Store {
+// BenchmarkFacIter_Invoke benchmarks the time spent invoking a factorial calculation.
+func BenchmarkFacIter_Invoke(b *testing.B) {
+	const in = 30
+	b.Run("interpreter", func(b *testing.B) {
+		store, err := newStoreForFacIterBench(interpreter.NewEngine())
+		if err != nil {
+			b.Fatal(err)
+		}
+		b.ResetTimer()
+		for i := 0; i < b.N; i++ {
+			if _, _, err = store.CallFunction("test", "fac-iter", in); err != nil {
+				b.Fatal(err)
+			}
+		}
+	})
+	b.Run("jit", func(b *testing.B) {
+		store, err := newStoreForFacIterBench(jit.NewEngine())
+		if err != nil {
+			b.Fatal(err)
+		}
+		b.ResetTimer()
+		for i := 0; i < b.N; i++ {
+			if _, _, err = store.CallFunction("test", "fac-iter", in); err != nil {
+				b.Fatal(err)
+			}
+		}
+	})
+	b.Run("wasmer-go", func(b *testing.B) {
+		store, instance, fn, err := newWasmerForFacIterBench()
+		if err != nil {
+			b.Fatal(err)
+		}
+		defer store.Close()
+		defer instance.Close()
+		b.ResetTimer()
+		for i := 0; i < b.N; i++ {
+			if _, err = fn(in); err != nil {
+				b.Fatal(err)
+			}
+		}
+	})
+	b.Run("wasmtime-go", func(b *testing.B) {
+		store, run, err := newWasmtimeForFacIterBench()
+		if err != nil {
+			b.Fatal(err)
+		}
+		b.ResetTimer()
+		for i := 0; i < b.N; i++ {
+			if _, err = run.Call(store, in); err != nil {
+				b.Fatal(err)
+			}
+		}
+	})
+}
+
+func newStoreForFacIterBench(engine wasm.Engine) (*wasm.Store, error) {
 	store := wasm.NewStore(engine)
-	buf, err := os.ReadFile("testdata/fac.wasm")
-	if err != nil {
-		panic(err)
-	}
-	mod, err := binaryFormat.DecodeModule(buf)
+	mod, err := binaryFormat.DecodeModule(facWasm)
 	if err != nil {
-		panic(err)
+		return nil, err
 	}
 	err = store.Instantiate(mod, "test")
 	if err != nil {
-		panic(err)
+		return nil, err
 	}
-	return store
+	return store, nil
 }
 
-func newWasmtimeForFacIterBench() (*wasmtime.Store, *wasmtime.Func) {
-	buf, err := os.ReadFile("testdata/fac.wasm")
+// newWasmerForFacIterBench returns the store and instance that scope the factorial function.
+// Note: these should be closed
+func newWasmerForFacIterBench() (*wasmer.Store, *wasmer.Instance, wasmer.NativeFunction, error) {
+	store := wasmer.NewStore(wasmer.NewEngine())
+	importObject := wasmer.NewImportObject()
+	module, err := wasmer.NewModule(store, facWasm)
+	if err != nil {
+		return nil, nil, nil, err
+	}
+	instance, err := wasmer.NewInstance(module, importObject)
+	if err != nil {
+		return nil, nil, nil, err
+	}
+	f, err := instance.Exports.GetFunction("fac-iter")
 	if err != nil {
-		panic(err)
+		return nil, nil, nil, err
+	}
+	if f == nil {
+		return nil, nil, nil, errors.New("not a function")
 	}
+	return store, instance, f, nil
+}
+
+func newWasmtimeForFacIterBench() (*wasmtime.Store, *wasmtime.Func, error) {
 	store := wasmtime.NewStore(wasmtime.NewEngine())
-	module, err := wasmtime.NewModule(store.Engine, buf)
+	module, err := wasmtime.NewModule(store.Engine, facWasm)
 	if err != nil {
-		panic(err)
+		return nil, nil, err
 	}
 
 	instance, err := wasmtime.NewInstance(store, module, nil)
 	if err != nil {
-		panic(err)
+		return nil, nil, err
 	}
 
 	run := instance.GetFunc(store, "fac-iter")
 	if run == nil {
-		panic("not a function")
+		return nil, nil, errors.New("not a function")
 	}
-	return store, run
+	return store, run, nil
 }
diff --git a/wasm/internal/example/example_test.go b/wasm/internal/example/example_test.go
index 1e7a1414..f2888fae 100644
--- a/wasm/internal/example/example_test.go
+++ b/wasm/internal/example/example_test.go
@@ -8,8 +8,9 @@ import (
 
 	"github.com/bytecodealliance/wasmtime-go"
 	"github.com/stretchr/testify/require"
+	"github.com/wasmerio/wasmer-go/wasmer"
 
-	wasm "github.com/tetratelabs/wazero/wasm"
+	"github.com/tetratelabs/wazero/wasm"
 	"github.com/tetratelabs/wazero/wasm/binary"
 	"github.com/tetratelabs/wazero/wasm/text"
 )
@@ -153,8 +154,20 @@ func BenchmarkCodecExample(b *testing.B) {
 			}
 		}
 	})
+	// Note: We don't know if wasmer.Wat2Wasm encodes the custom name section or not.
+	// Note: wasmer.Wat2Wasm calls wasmer via CGO which is eventually implemented by wasm-tools
+	b.Run("wat2wasm vs wasmer.Wat2Wasm", func(b *testing.B) {
+		b.ReportAllocs()
+		for i := 0; i < b.N; i++ {
+			_, err := wasmer.Wat2Wasm(string(exampleText))
+			if err != nil {
+				panic(err)
+			}
+		}
+	})
 	// Note: We don't know if wasmtime.Wat2Wasm encodes the custom name section or not.
-	b.Run("wat2wasm via wasmtime.Wat2Wasm", func(b *testing.B) {
+	// Note: wasmtime.Wat2Wasm calls wasmtime via CGO which is eventually implemented by wasm-tools
+	b.Run("wat2wasm vs wasmtime.Wat2Wasm", func(b *testing.B) {
 		b.ReportAllocs()
 		for i := 0; i < b.N; i++ {
 			_, err := wasmtime.Wat2Wasm(string(exampleText))