Repository: tetratelabs/wazero (go)

Issue:
Deadlock in compilation cache
**Describe the bug**
Use of the compilation cache between wazero versions seems to be able to cause a deadlock when the cache is cleaned.

**To Reproduce**
```
$ go mod init example.com/wazero-dl
go: creating new go.mod: module example.com/wazero-dl
$ cat <<EOF > main.go
package main

import (
        "context"
        "log"
        "os"
        "time"

        "github.com/tetratelabs/wazero"
        "github.com/tetratelabs/wazero/experimental"
)

func main() {
        bin, err := os.ReadFile(os.Args[1])
        if err != nil {
                log.Panicln(err)
        }

        ctx, err := experimental.WithCompilationCacheDirName(context.Background(), ".build")
        if err != nil {
                log.Panicln(err)
        }
        r := wazero.NewRuntime(ctx)
        defer r.Close(ctx) // This closes everything this Runtime created.

        start := time.Now()
        if _, err = r.CompileModule(ctx, bin); err != nil {
                log.Panicln(err)
        }
        log.Printf("CompileModule took %dms", time.Since(start).Milliseconds())
}
EOF
$ go get github.com/tetratelabs/wazero@3f578ddac3897b1435fbb1fe7e807676f062e4b1
go: added github.com/tetratelabs/wazero v1.0.0-pre.5.0.20221229073945-3f578ddac389
$ echo '(module (func (export "main") (result i32) i32.const 42 return))' | wat2wasm - -o simple.wasm
$ go run . simple.wasm
2022/12/29 15:16:56 CompileModule took 0ms
$ go get github.com/tetratelabs/wazero@62be68dfc5f477d766d9dcfe422dcfc498c7bbfd
go: downloading github.com/tetratelabs/wazero v1.0.0-pre.5.0.20221229092947-62be68dfc5f4
go: upgraded github.com/tetratelabs/wazero v1.0.0-pre.5.0.20221229073945-3f578ddac389 => v1.0.0-pre.5.0.20221229092947-62be68dfc5f4
$ go run . simple.wasm
fatal error: all goroutines are asleep - deadlock!

goroutine 1 [semacquire]:
sync.runtime_SemacquireMutex(0x104425620?, 0xa0?, 0x1400013db68?)
        /Users/robbert/.gimme/versions/go1.19.3.darwin.arm64/src/runtime/sema.go:77 +0x24
sync.(*RWMutex).Lock(0x101400013dba8?)
        /Users/robbert/.gimme/versions/go1.19.3.darwin.arm64/src/sync/rwmutex.go:152 +0x100
github.com/tetratelabs/wazero/internal/compilationcache.(*fileCache).Delete(0x14000115170, {0xc1, 0x83, 0xb6, 0x11, 0xda, 0x74, 0xfe, 0xc0, 0x46, ...})
        /Users/robbert/go/pkg/mod/github.com/tetratelabs/wazero@v1.0.0-pre.5.0.20221229092947-62be68dfc5f4/internal/compilationcache/file_cache.go:92 +0x3c
github.com/tetratelabs/wazero/internal/engine/compiler.(*engine).getCodesFromCache(0x1400010c0a0, 0x14000102680)
        /Users/robbert/go/pkg/mod/github.com/tetratelabs/wazero@v1.0.0-pre.5.0.20221229092947-62be68dfc5f4/internal/engine/compiler/engine_cache.go:84 +0x18c
github.com/tetratelabs/wazero/internal/engine/compiler.(*engine).getCodes(0x1400013dd58?, 0x1043312f4?)
        /Users/robbert/go/pkg/mod/github.com/tetratelabs/wazero@v1.0.0-pre.5.0.20221229092947-62be68dfc5f4/internal/engine/compiler/engine_cache.go:35 +0x34
github.com/tetratelabs/wazero/internal/engine/compiler.(*engine).CompileModule(0x1400010c0a0, {0x1044482b8, 0x14000115110}, 0x14000102680, {0x0, 0x0, 0x200?})
        /Users/robbert/go/pkg/mod/github.com/tetratelabs/wazero@v1.0.0-pre.5.0.20221229092947-62be68dfc5f4/internal/engine/compiler/engine.go:488 +0x3c
github.com/tetratelabs/wazero.(*runtime).CompileModule(0x140001260c0, {0x1044482b8, 0x14000115110}, {0x14000164000, 0x26, 0x200})
        /Users/robbert/go/pkg/mod/github.com/tetratelabs/wazero@v1.0.0-pre.5.0.20221229092947-62be68dfc5f4/runtime.go:200 +0x258
main.main()
        $/main.go:27 +0x160
exit status 2
```
**Expected behavior**
No deadlock.

**Screenshots**
N/a

**Environment (please complete the relevant information):**
- Go version: 1.19.3
- wazero Version: see repro, experienced on other commits too (se below).
- Host architecture: arm64
- Runtime mode: n/a

**Additional context**
Narrowed this down to a simple example from what I'd previously mentioned in https://github.com/tetratelabs/wazero/issues/939#issuecomment-1358589231. I could reproduce it it for the sha's mentioned there too but for the repo I used HEAD & HEAD~ to make sure this was still an issue.

Pull request description:
cache: fixes deadlock in version mismatch
Fixes #977 

Patch:
diff --git a/internal/engine/compiler/engine_cache.go b/internal/engine/compiler/engine_cache.go
index 0e9ab719..18f14279 100644
--- a/internal/engine/compiler/engine_cache.go
+++ b/internal/engine/compiler/engine_cache.go
@@ -71,11 +71,11 @@ func (e *engine) getCodesFromCache(module *wasm.Module) (codes []*code, hit bool
 	if !hit || err != nil {
 		return
 	}
-	defer cached.Close()
 
 	// Otherwise, we hit the cache on external cache.
 	// We retrieve *code structures from `cached`.
 	var staleCache bool
+	// Note: cached.Close is ensured to be called in deserializeCodes.
 	codes, staleCache, err = deserializeCodes(e.wazeroVersion, cached)
 	if err != nil {
 		hit = false
@@ -114,7 +114,8 @@ func serializeCodes(wazeroVersion string, codes []*code) io.Reader {
 	return bytes.NewReader(buf.Bytes())
 }
 
-func deserializeCodes(wazeroVersion string, reader io.Reader) (codes []*code, staleCache bool, err error) {
+func deserializeCodes(wazeroVersion string, reader io.ReadCloser) (codes []*code, staleCache bool, err error) {
+	defer reader.Close()
 	cacheHeaderSize := len(wazeroMagic) + 1 /* version size */ + len(wazeroVersion) + 4 /* number of functions */
 
 	// Read the header before the native code.


Test patch:
diff --git a/internal/engine/compiler/engine_cache_test.go b/internal/engine/compiler/engine_cache_test.go
index 8ef85dd6..47290f00 100644
--- a/internal/engine/compiler/engine_cache_test.go
+++ b/internal/engine/compiler/engine_cache_test.go
@@ -2,22 +2,23 @@ package compiler
 
 import (
 	"bytes"
+	"context"
 	"crypto/sha256"
 	"encoding/binary"
 	"errors"
-	"fmt"
 	"io"
 	"math"
 	"testing"
 	"testing/iotest"
 
+	"github.com/tetratelabs/wazero/internal/compilationcache"
 	"github.com/tetratelabs/wazero/internal/testing/require"
 	"github.com/tetratelabs/wazero/internal/u32"
 	"github.com/tetratelabs/wazero/internal/u64"
 	"github.com/tetratelabs/wazero/internal/wasm"
 )
 
-var testVersion string
+var testVersion = ""
 
 func concat(ins ...[]byte) (ret []byte) {
 	for _, in := range ins {
@@ -199,7 +200,7 @@ func TestDeserializeCodes(t *testing.T) {
 	for _, tc := range tests {
 		tc := tc
 		t.Run(tc.name, func(t *testing.T) {
-			codes, staleCache, err := deserializeCodes(testVersion, bytes.NewReader(tc.in))
+			codes, staleCache, err := deserializeCodes(testVersion, io.NopCloser(bytes.NewReader(tc.in)))
 			if tc.expErr != "" {
 				require.EqualError(t, err, tc.expErr)
 			} else {
@@ -230,7 +231,7 @@ func TestEngine_getCodesFromCache(t *testing.T) {
 
 	tests := []struct {
 		name       string
-		ext        *testCache
+		ext        map[wasm.ModuleID][]byte
 		key        wasm.ModuleID
 		isHostMod  bool
 		expCodes   []*code
@@ -241,38 +242,38 @@ func TestEngine_getCodesFromCache(t *testing.T) {
 		{name: "extern cache not given"},
 		{
 			name: "not hit",
-			ext:  &testCache{caches: map[wasm.ModuleID][]byte{}},
+			ext:  map[wasm.ModuleID][]byte{},
 		},
 		{
 			name:      "host module",
-			ext:       &testCache{caches: map[wasm.ModuleID][]byte{{}: valid}},
+			ext:       map[wasm.ModuleID][]byte{{}: valid},
 			isHostMod: true,
 		},
 		{
 			name:   "error in Cache.Get",
-			ext:    &testCache{caches: map[wasm.ModuleID][]byte{{}: {}}},
-			expErr: "some error from extern cache",
+			ext:    map[wasm.ModuleID][]byte{{}: {}},
+			expErr: "compilationcache: error reading header: EOF",
 		},
 		{
 			name:   "error in deserialization",
-			ext:    &testCache{caches: map[wasm.ModuleID][]byte{{}: {1, 2, 3}}},
+			ext:    map[wasm.ModuleID][]byte{{}: {1, 2, 3}},
 			expErr: "compilationcache: invalid header length: 3",
 		},
 		{
 			name: "stale cache",
-			ext: &testCache{caches: map[wasm.ModuleID][]byte{{}: concat(
+			ext: map[wasm.ModuleID][]byte{{}: concat(
 				[]byte(wazeroMagic),
 				[]byte{byte(len("1233123.1.1"))},
 				[]byte("1233123.1.1"),
 				u32.LeBytes(1), // number of functions.
-			)}},
+			)},
 			expDeleted: true,
 		},
 		{
 			name: "hit",
-			ext: &testCache{caches: map[wasm.ModuleID][]byte{
+			ext: map[wasm.ModuleID][]byte{
 				{}: valid,
-			}},
+			},
 			expHit: true,
 			expCodes: []*code{
 				{stackPointerCeil: 12345, codeSegment: []byte{1, 2, 3, 4, 5}, indexInModule: 0},
@@ -291,7 +292,14 @@ func TestEngine_getCodesFromCache(t *testing.T) {
 
 			e := engine{}
 			if tc.ext != nil {
-				e.Cache = tc.ext
+				tmp := t.TempDir()
+				e.Cache = compilationcache.NewFileCache(
+					context.WithValue(context.Background(), compilationcache.FileCachePathKey{}, tmp),
+				)
+				for key, value := range tc.ext {
+					err := e.Cache.Add(key, bytes.NewReader(value))
+					require.NoError(t, err)
+				}
 			}
 
 			codes, hit, err := e.getCodesFromCache(m)
@@ -304,8 +312,10 @@ func TestEngine_getCodesFromCache(t *testing.T) {
 			require.Equal(t, tc.expHit, hit)
 			require.Equal(t, tc.expCodes, codes)
 
-			if tc.expDeleted {
-				require.Equal(t, tc.ext.deleted, tc.key)
+			if tc.ext != nil && tc.expDeleted {
+				_, hit, err := e.Cache.Get(tc.key)
+				require.NoError(t, err)
+				require.False(t, hit)
 			}
 		})
 	}
@@ -318,27 +328,32 @@ func TestEngine_addCodesToCache(t *testing.T) {
 		require.NoError(t, err)
 	})
 	t.Run("host module", func(t *testing.T) {
-		tc := &testCache{caches: map[wasm.ModuleID][]byte{}}
+		tc := compilationcache.NewFileCache(context.WithValue(context.Background(),
+			compilationcache.FileCachePathKey{}, t.TempDir()))
 		e := engine{Cache: tc}
 		codes := []*code{{stackPointerCeil: 123, codeSegment: []byte{1, 2, 3}}}
 		m := &wasm.Module{ID: sha256.Sum256(nil), IsHostModule: true} // Host module!
 		err := e.addCodesToCache(m, codes)
 		require.NoError(t, err)
 		// Check the host module not cached.
-		content, ok := tc.caches[m.ID]
-		require.False(t, ok)
-		require.Nil(t, content)
+		_, hit, err := tc.Get(m.ID)
+		require.NoError(t, err)
+		require.False(t, hit)
 	})
 	t.Run("add", func(t *testing.T) {
-		ext := &testCache{caches: map[wasm.ModuleID][]byte{}}
-		e := engine{Cache: ext}
+		tc := compilationcache.NewFileCache(context.WithValue(context.Background(),
+			compilationcache.FileCachePathKey{}, t.TempDir()))
+		e := engine{Cache: tc}
 		m := &wasm.Module{}
 		codes := []*code{{stackPointerCeil: 123, codeSegment: []byte{1, 2, 3}}}
 		err := e.addCodesToCache(m, codes)
 		require.NoError(t, err)
 
-		content, ok := ext.caches[m.ID]
+		content, ok, err := tc.Get(m.ID)
+		require.NoError(t, err)
 		require.True(t, ok)
+		actual, err := io.ReadAll(content)
+		require.NoError(t, err)
 		require.Equal(t, concat(
 			[]byte(wazeroMagic),
 			[]byte{byte(len(testVersion))},
@@ -347,7 +362,8 @@ func TestEngine_addCodesToCache(t *testing.T) {
 			u64.LeBytes(123), // stack pointer ceil.
 			u64.LeBytes(3),   // length of code.
 			[]byte{1, 2, 3},  // code.
-		), content)
+		), actual)
+		require.NoError(t, content.Close())
 	})
 }
 
@@ -422,43 +438,3 @@ func Test_readUint64_errors(t *testing.T) {
 		})
 	}
 }
-
-// testCache implements compilationcache.Cache
-type testCache struct {
-	caches  map[wasm.ModuleID][]byte
-	deleted wasm.ModuleID
-}
-
-// Get implements compilationcache.Cache Get
-func (tc *testCache) Get(key wasm.ModuleID) (content io.ReadCloser, ok bool, err error) {
-	var raw []byte
-	raw, ok = tc.caches[key]
-	if !ok {
-		return
-	}
-
-	if len(raw) == 0 {
-		ok = false
-		err = fmt.Errorf("some error from extern cache")
-		return
-	}
-
-	content = io.NopCloser(bytes.NewReader(raw))
-	return
-}
-
-// Add implements compilationcache.Cache Add
-func (tc *testCache) Add(key wasm.ModuleID, content io.Reader) (err error) {
-	raw, err := io.ReadAll(content)
-	if err != nil {
-		return err
-	}
-	tc.caches[key] = raw
-	return
-}
-
-// Delete implements compilationcache.Cache Delete
-func (tc *testCache) Delete(key wasm.ModuleID) (err error) {
-	tc.deleted = key
-	return
-}