Repository: buchgr/bazel-remote
Pull request #559 (2022-06-28 22:24:32)

respin glukasik's cgo zstd PR
This is a rebased version of #485.


Patch:
diff --git a/.bazelci/presubmit.yml b/.bazelci/presubmit.yml
index 8eec421..6eef78e 100644
--- a/.bazelci/presubmit.yml
+++ b/.bazelci/presubmit.yml
@@ -7,13 +7,15 @@ tasks:
   ubuntu1804:
     platform: ubuntu1804
     build_targets:
-      - "..."
+      - "//:bazel-remote"
+      - "//:bazel-remote-image"
     test_targets:
       - "..."
   ubuntu2004:
     platform: ubuntu2004
     build_targets:
-      - "..."
+      - "//:bazel-remote"
+      - "//:bazel-remote-image"
     test_targets:
       - "..."
   macos:
@@ -53,13 +55,20 @@ tasks:
       - ".bazelci/buildkite-install-go.sh"
       - "echo +++ Running go test with race detector"
       - "PATH=$HOME/go/bin:$PATH go test -race ./..."
-  end_to_end_test:
+  end_to_end_test_go:
     platform: ubuntu2004
-    name: "end-to-end test"
+    name: "end-to-end test - pure go zstd"
     shell_commands:
       - ".bazelci/buildkite-install-go.sh"
-      - "echo +++ Run end-to-end system test"
+      - "echo +++ Run end-to-end system test - pure go zstd"
       - "PATH=$HOME/go/bin:$PATH timeout 30m .bazelci/system-test.sh"
+  end_to_end_test_cgo:
+    platform: ubuntu2004
+    name: "end-to-end test - cgo zstd"
+    shell_commands:
+      - ".bazelci/buildkite-install-go.sh"
+      - "echo +++ Run end-to-end system test - cgo zstd"
+      - "PATH=$HOME/go/bin:$PATH EXTRA_FLAGS=\"--zstd_implementation=cgo\" timeout 30m .bazelci/system-test.sh"
   tls_tests:
     platform: ubuntu2004
     name: "TLS tests"
diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
index b9a6150..21685b2 100644
--- a/.github/workflows/release.yml
+++ b/.github/workflows/release.yml
@@ -8,7 +8,7 @@ on:
 jobs:
   linux:
     name: create linux binaries
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-18.04
     steps:
       - name: checkout code
         uses: actions/checkout@v2
diff --git a/BUILD.bazel b/BUILD.bazel
index 23f3134..aa4c529 100644
--- a/BUILD.bazel
+++ b/BUILD.bazel
@@ -46,8 +46,8 @@ go_library(
 go_binary(
     name = "bazel-remote",
     embed = [":go_default_library"],
-    pure = "on",
-    static = "on",
+    pure = "off",
+    static = "off",
     visibility = ["//visibility:public"],
     x_defs = {"main.gitCommit": "{STABLE_GIT_COMMIT}"},
 )
@@ -57,8 +57,8 @@ go_binary(
     embed = [":go_default_library"],
     goarch = "amd64",
     goos = "linux",
-    pure = "on",
-    static = "on",
+    pure = "off",
+    static = "off",
     visibility = ["//visibility:public"],
     x_defs = {"main.gitCommit": "{STABLE_GIT_COMMIT}"},
 )
@@ -104,7 +104,7 @@ go_image(
     embed = [":go_default_library"],
     goarch = "amd64",
     goos = "linux",
-    pure = "on",
+    pure = "off",
     static = "on",
     visibility = ["//visibility:private"],
     x_defs = {"main.gitCommit": "{STABLE_GIT_COMMIT}"},
@@ -115,7 +115,7 @@ go_image(
     embed = [":go_default_library"],
     goarch = "arm64",
     goos = "linux",
-    pure = "on",
+    pure = "off",
     static = "on",
     visibility = ["//visibility:private"],
     x_defs = {"main.gitCommit": "{STABLE_GIT_COMMIT}"},
diff --git a/README.md b/README.md
index 610c7b8..8b292cd 100644
--- a/README.md
+++ b/README.md
@@ -148,6 +148,9 @@ OPTIONS:
    --storage_mode value Which format to store CAS blobs in. Must be one of
       "zstd" or "uncompressed". (default: "zstd") [$BAZEL_REMOTE_STORAGE_MODE]
 
+   --zstd_implementation value ZSTD implementation to use. Must be one of
+      "go" or "cgo". (default: "go") [$BAZEL_REMOTE_ZSTD_IMPLEMENTATION]
+
    --http_address value Address specification for the HTTP server listener,
       formatted either as [host]:port for TCP or unix://path.sock for Unix
       domain sockets. [$BAZEL_REMOTE_HTTP_ADDRESS]
diff --git a/cache/disk/BUILD.bazel b/cache/disk/BUILD.bazel
index 7250d8e..3465482 100644
--- a/cache/disk/BUILD.bazel
+++ b/cache/disk/BUILD.bazel
@@ -14,6 +14,7 @@ go_library(
     deps = [
         "//cache:go_default_library",
         "//cache/disk/casblob:go_default_library",
+        "//cache/disk/zstdimpl:go_default_library",
         "//genproto/build/bazel/remote/execution/v2:go_default_library",
         "//utils/tempfile:go_default_library",
         "//utils/validate:go_default_library",
diff --git a/cache/disk/casblob/BUILD.bazel b/cache/disk/casblob/BUILD.bazel
index 7036b4a..98be79e 100644
--- a/cache/disk/casblob/BUILD.bazel
+++ b/cache/disk/casblob/BUILD.bazel
@@ -5,11 +5,7 @@ go_library(
     srcs = ["casblob.go"],
     importpath = "github.com/buchgr/bazel-remote/cache/disk/casblob",
     visibility = ["//visibility:public"],
-    deps = [
-        "//utils/zstdpool:go_default_library",
-        "@com_github_klauspost_compress//zstd:go_default_library",
-        "@com_github_mostynb_zstdpool_syncpool//:go_default_library",
-    ],
+    deps = ["//cache/disk/zstdimpl:go_default_library"],
 )
 
 go_test(
diff --git a/cache/disk/casblob/casblob.go b/cache/disk/casblob/casblob.go
index a5f092e..367a0fc 100644
--- a/cache/disk/casblob/casblob.go
+++ b/cache/disk/casblob/casblob.go
@@ -12,9 +12,7 @@ import (
 	"log"
 	"os"
 
-	"github.com/buchgr/bazel-remote/utils/zstdpool"
-	"github.com/klauspost/compress/zstd"
-	syncpool "github.com/mostynb/zstdpool-syncpool"
+	"github.com/buchgr/bazel-remote/cache/disk/zstdimpl"
 )
 
 type CompressionType uint8
@@ -24,17 +22,6 @@ const (
 	Zstandard CompressionType = 1
 )
 
-var zstdFastestLevel = zstd.WithEncoderLevel(zstd.SpeedFastest)
-
-var encoder, _ = zstd.NewWriter(nil, zstdFastestLevel) // TODO: raise WithEncoderConcurrency ?
-var decoder, _ = zstd.NewReader(nil)                   // TODO: raise WithDecoderConcurrency ?
-
-var encoderPool = zstdpool.GetEncoderPool()
-var decoderPool = zstdpool.GetDecoderPool()
-
-var errDecoderPoolFail error = errors.New("failed to get DecoderWrapper from pool")
-var errEncoderPoolFail error = errors.New("failed to get EncoderWrapper from pool")
-
 const defaultChunkSize = 1024 * 1024 * 1 // 1M
 
 // 4 bytes, to be written to disk in little-endian format.
@@ -85,8 +72,9 @@ type readCloserWrapper struct {
 
 	rdr io.Reader // Read from this, not from decoder or file.
 
-	decoder *syncpool.DecoderWrapper // Might be nil.
-	file    *os.File
+	rdrCloser io.Closer // Might be nil.
+
+	file *os.File
 }
 
 var errWrongMagicNum = errors.New("expected magic number not found")
@@ -227,7 +215,7 @@ func (m *multiReadCloser) Close() error {
 // must close the returned io.ReadCloser if it is non-nil. Doing so
 // will automatically close f. If there is an error f will be closed, the caller
 // does not need to do so.
-func GetUncompressedReadCloser(f *os.File, expectedSize int64, offset int64) (io.ReadCloser, error) {
+func GetUncompressedReadCloser(zstd zstdimpl.ZstdImpl, f *os.File, expectedSize int64, offset int64) (io.ReadCloser, error) {
 	h, err := readHeader(f)
 	if err != nil {
 		f.Close()
@@ -274,22 +262,17 @@ func GetUncompressedReadCloser(f *os.File, expectedSize int64, offset int64) (io
 		}
 	}
 	if remainder == 0 {
-		z, ok := decoderPool.Get().(*syncpool.DecoderWrapper)
-		if !ok {
-			f.Close()
-			return nil, err
-		}
-		err := z.Reset(f)
+		dec, err := zstd.GetDecoder(f)
 		if err != nil {
 			f.Close()
 			return nil, err
 		}
 
 		return &readCloserWrapper{
-			header:  h,
-			rdr:     z,
-			decoder: z,
-			file:    f,
+			header:    h,
+			rdr:       dec,
+			rdrCloser: dec,
+			file:      f,
 		}, nil
 	}
 
@@ -300,7 +283,7 @@ func GetUncompressedReadCloser(f *os.File, expectedSize int64, offset int64) (io
 		return nil, err
 	}
 
-	uncompressedFirstChunk, err := decoder.DecodeAll(compressedFirstChunk, nil)
+	uncompressedFirstChunk, err := zstd.DecodeAll(compressedFirstChunk)
 	if err != nil {
 		f.Close()
 		return nil, err
@@ -313,26 +296,19 @@ func GetUncompressedReadCloser(f *os.File, expectedSize int64, offset int64) (io
 		return ioutil.NopCloser(r), nil
 	}
 
-	z, ok := decoderPool.Get().(*syncpool.DecoderWrapper)
-	if !ok {
-		f.Close()
-		return nil, errDecoderPoolFail
-	}
-
-	err = z.Reset(f)
+	z, err := zstd.GetDecoder(f)
 	if err != nil {
 		f.Close()
-		decoderPool.Put(z)
-		return nil, fmt.Errorf("Failed to setup zstd decoder: %w", err)
+		return nil, err
 	}
 
 	br := bytes.NewReader(uncompressedFirstChunk[remainder:])
 
 	return &readCloserWrapper{
-		header:  h,
-		rdr:     io.MultiReader(br, z),
-		decoder: z,
-		file:    f,
+		header:    h,
+		rdr:       io.MultiReader(br, z),
+		rdrCloser: z,
+		file:      f,
 	}, nil
 }
 
@@ -340,7 +316,7 @@ func GetUncompressedReadCloser(f *os.File, expectedSize int64, offset int64) (io
 // caller must close the returned io.ReadCloser if it is non-nil. Doing so
 // will automatically close f. If there is an error f will be closed, the caller
 // does not need to do so.
-func GetZstdReadCloser(f *os.File, expectedSize int64, offset int64) (io.ReadCloser, error) {
+func GetZstdReadCloser(zstd zstdimpl.ZstdImpl, f *os.File, expectedSize int64, offset int64) (io.ReadCloser, error) {
 
 	h, err := readHeader(f)
 	if err != nil {
@@ -366,7 +342,7 @@ func GetZstdReadCloser(f *os.File, expectedSize int64, offset int64) (io.ReadClo
 			}
 		}
 
-		return GetLegacyZstdReadCloser(f)
+		return GetLegacyZstdReadCloser(zstd, f)
 	}
 
 	if h.compression != Zstandard {
@@ -399,14 +375,14 @@ func GetZstdReadCloser(f *os.File, expectedSize int64, offset int64) (io.ReadClo
 		return nil, err
 	}
 
-	uncompressedFirstChunk, err := decoder.DecodeAll(compressedFirstChunk, nil)
+	uncompressedFirstChunk, err := zstd.DecodeAll(compressedFirstChunk)
 	if err != nil {
 		f.Close()
 		return nil, err
 	}
 
 	chunkToRecompress := uncompressedFirstChunk[remainder:]
-	recompressedChunk := encoder.EncodeAll(chunkToRecompress, nil)
+	recompressedChunk := zstd.EncodeAll(chunkToRecompress)
 
 	br := bytes.NewReader(recompressedChunk)
 	if chunkNum == int64(len(h.chunkOffsets)-2) {
@@ -423,15 +399,15 @@ func GetZstdReadCloser(f *os.File, expectedSize int64, offset int64) (io.ReadClo
 
 // GetLegacyZstdReadCloser returns an io.ReadCloser that provides
 // zstandard-compressed data from an uncompressed file.
-func GetLegacyZstdReadCloser(f *os.File) (io.ReadCloser, error) {
-	enc, ok := encoderPool.Get().(*syncpool.EncoderWrapper)
-	if !ok {
-		f.Close()
-		return nil, errEncoderPoolFail
-	}
+func GetLegacyZstdReadCloser(zstd zstdimpl.ZstdImpl, f *os.File) (io.ReadCloser, error) {
 
 	pr, pw := io.Pipe()
-	enc.Reset(pw)
+
+	enc, err := zstd.GetEncoder(f)
+	if err != nil {
+		_ = f.Close()
+		return nil, err
+	}
 
 	go func() {
 		// Read from the file, write to enc.
@@ -445,8 +421,12 @@ func GetLegacyZstdReadCloser(f *os.File) (io.ReadCloser, error) {
 			_ = pw.CloseWithError(err)
 		}
 
-		encoderPool.Put(enc)
-		f.Close()
+		err = enc.Close()
+		if err != nil {
+			log.Println("Error while closing encoder:", err)
+			_ = pw.CloseWithError(err)
+		}
+		_ = f.Close()
 	}()
 
 	return pr, nil
@@ -493,9 +473,9 @@ func (b *readCloserWrapper) Read(p []byte) (int, error) {
 }
 
 func (b *readCloserWrapper) Close() error {
-	if b.decoder != nil {
-		b.decoder.Close()
-		b.decoder = nil
+	if b.rdrCloser != nil {
+		_ = b.rdrCloser.Close()
+		b.rdrCloser = nil
 	}
 
 	if b.file == nil {
@@ -510,7 +490,7 @@ func (b *readCloserWrapper) Close() error {
 
 // Read from r and write to f, using CompressionType t.
 // Return the size on disk or an error if something went wrong.
-func WriteAndClose(r io.Reader, f *os.File, t CompressionType, hash string, size int64) (int64, error) {
+func WriteAndClose(zstd zstdimpl.ZstdImpl, r io.Reader, f *os.File, t CompressionType, hash string, size int64) (int64, error) {
 	var err error
 	defer f.Close()
 
@@ -595,7 +575,7 @@ func WriteAndClose(r io.Reader, f *os.File, t CompressionType, hash string, size
 			return -1, err
 		}
 
-		compressedChunk := encoder.EncodeAll(uncompressedChunk[0:chunkEnd], nil)
+		compressedChunk := zstd.EncodeAll(uncompressedChunk[0:chunkEnd])
 
 		hasher.Write(uncompressedChunk[0:chunkEnd])
 
diff --git a/cache/disk/disk.go b/cache/disk/disk.go
index 3ceff01..ee30397 100644
--- a/cache/disk/disk.go
+++ b/cache/disk/disk.go
@@ -24,6 +24,7 @@ import (
 
 	"github.com/buchgr/bazel-remote/cache"
 	"github.com/buchgr/bazel-remote/cache/disk/casblob"
+	"github.com/buchgr/bazel-remote/cache/disk/zstdimpl"
 	"github.com/buchgr/bazel-remote/utils/tempfile"
 	"github.com/buchgr/bazel-remote/utils/validate"
 
@@ -76,6 +77,7 @@ type diskCache struct {
 	dir              string
 	proxy            cache.Proxy
 	storageMode      casblob.CompressionType
+	zstd             zstdimpl.ZstdImpl
 	maxBlobSize      int64
 	maxProxyBlobSize int64
 	accessLogger     *log.Logger
@@ -140,11 +142,17 @@ func New(dir string, maxSizeBytes int64, opts ...Option) (Cache, error) {
 	}
 	log.Printf("Limiting concurrent file removals to %d\n", semaphoreWeight)
 
+	zi, err := zstdimpl.Get("go")
+	if err != nil {
+		return nil, err
+	}
+
 	c := diskCache{
 		dir: dir,
 
 		// Not using config here, to avoid test import cycles.
 		storageMode:      casblob.Zstandard,
+		zstd:             zi,
 		maxBlobSize:      math.MaxInt64,
 		maxProxyBlobSize: math.MaxInt64,
 
@@ -735,7 +743,7 @@ func (c *diskCache) writeAndCloseFile(r io.Reader, kind cache.EntryKind, hash st
 	var sizeOnDisk int64
 
 	if kind == cache.CAS && c.storageMode != casblob.Identity {
-		sizeOnDisk, err = casblob.WriteAndClose(r, f, c.storageMode, hash, size)
+		sizeOnDisk, err = casblob.WriteAndClose(c.zstd, r, f, c.storageMode, hash, size)
 		if err != nil {
 			return -1, err
 		}
@@ -842,16 +850,16 @@ func (c *diskCache) availableOrTryProxy(kind cache.EntryKind, hash string, size
 					// The file is uncompressed, without a casblob header.
 					_, err = f.Seek(offset, io.SeekStart)
 					if zstd && err == nil {
-						rc, err = casblob.GetLegacyZstdReadCloser(f)
+						rc, err = casblob.GetLegacyZstdReadCloser(c.zstd, f)
 					} else if err == nil {
 						rc = f
 					}
 				} else {
 					// The file is compressed.
 					if zstd {
-						rc, err = casblob.GetZstdReadCloser(f, size, offset)
+						rc, err = casblob.GetZstdReadCloser(c.zstd, f, size, offset)
 					} else {
-						rc, err = casblob.GetUncompressedReadCloser(f, size, offset)
+						rc, err = casblob.GetUncompressedReadCloser(c.zstd, f, size, offset)
 					}
 				}
 
@@ -1054,15 +1062,15 @@ func (c *diskCache) get(ctx context.Context, kind cache.EntryKind, hash string,
 		}
 
 		if zstd {
-			rc, err = casblob.GetLegacyZstdReadCloser(rcf)
+			rc, err = casblob.GetLegacyZstdReadCloser(c.zstd, rcf)
 		} else {
 			rc = rcf
 		}
 	} else { // Compressed CAS blob.
 		if zstd {
-			rc, err = casblob.GetZstdReadCloser(rcf, foundSize, offset)
+			rc, err = casblob.GetZstdReadCloser(c.zstd, rcf, foundSize, offset)
 		} else {
-			rc, err = casblob.GetUncompressedReadCloser(rcf, foundSize, offset)
+			rc, err = casblob.GetUncompressedReadCloser(c.zstd, rcf, foundSize, offset)
 		}
 	}
 	if err != nil {
diff --git a/cache/disk/options.go b/cache/disk/options.go
index bf103e0..2533923 100644
--- a/cache/disk/options.go
+++ b/cache/disk/options.go
@@ -6,6 +6,7 @@ import (
 
 	"github.com/buchgr/bazel-remote/cache"
 	"github.com/buchgr/bazel-remote/cache/disk/casblob"
+	"github.com/buchgr/bazel-remote/cache/disk/zstdimpl"
 
 	"github.com/prometheus/client_golang/prometheus"
 )
@@ -31,6 +32,14 @@ func WithStorageMode(mode string) Option {
 	}
 }
 
+func WithZstdImplementation(impl string) Option {
+	return func(c *CacheConfig) error {
+		var err error
+		c.diskCache.zstd, err = zstdimpl.Get(impl)
+		return err
+	}
+}
+
 func WithMaxBlobSize(size int64) Option {
 	return func(c *CacheConfig) error {
 		if size <= 0 {
diff --git a/cache/disk/zstdimpl/BUILD.bazel b/cache/disk/zstdimpl/BUILD.bazel
new file mode 100644
index 0000000..e4b2d25
--- /dev/null
+++ b/cache/disk/zstdimpl/BUILD.bazel
@@ -0,0 +1,26 @@
+load("@io_bazel_rules_go//go:def.bzl", "go_library")
+
+go_library(
+    name = "go_default_library",
+    srcs = [
+        "gozstd.go",
+        "zstdimpl.go",
+    ] + select({
+        "@io_bazel_rules_go//go/toolchain:cgo_on": [
+            "cgozstd.go",
+        ],
+        "//conditions:default": [],
+    }),
+    importpath = "github.com/buchgr/bazel-remote/cache/disk/zstdimpl",
+    visibility = ["//visibility:public"],
+    deps = [
+        "//utils/zstdpool:go_default_library",
+        "@com_github_klauspost_compress//zstd:go_default_library",
+        "@com_github_mostynb_zstdpool_syncpool//:go_default_library",
+    ] + select({
+        "@io_bazel_rules_go//go/toolchain:cgo_on": [
+            "@com_github_valyala_gozstd//:go_default_library",
+        ],
+        "//conditions:default": [],
+    }),
+)
diff --git a/cache/disk/zstdimpl/cgozstd.go b/cache/disk/zstdimpl/cgozstd.go
new file mode 100644
index 0000000..55b773c
--- /dev/null
+++ b/cache/disk/zstdimpl/cgozstd.go
@@ -0,0 +1,97 @@
+//go:build cgo
+// +build cgo
+
+package zstdimpl
+
+import (
+	"io"
+	"runtime"
+	"sync"
+
+	"github.com/valyala/gozstd"
+)
+
+const compressionLevel = 1
+
+type cgoZstd struct{}
+
+func init() {
+	register("cgo", cgoZstd{})
+}
+
+func (cgoZstd) GetDecoder(in io.ReadCloser) (io.ReadCloser, error) {
+	r := readerPool.Get().(*readerWrapper)
+	r.Reset(in, nil)
+	return &putReaderToPoolOnClose{r}, nil
+}
+
+func (cgoZstd) GetEncoder(out io.WriteCloser) (zstdEncoder, error) {
+	w := writerPool.Get().(*writerWrapper)
+	w.Reset(out, nil, compressionLevel)
+	return &putWriterToPoolOnClose{w}, nil
+}
+
+func (cgoZstd) DecodeAll(in []byte) ([]byte, error) {
+	return gozstd.Decompress(nil, in)
+}
+
+func (cgoZstd) EncodeAll(in []byte) []byte {
+	return gozstd.CompressLevel(nil, in, compressionLevel)
+}
+
+// -- Reader pool
+var readerPool = &sync.Pool{
+	New: newReader,
+}
+
+type readerWrapper struct {
+	*gozstd.Reader
+}
+
+func newReader() interface{} {
+	r := &readerWrapper{gozstd.NewReader(nil)}
+	runtime.SetFinalizer(r, releaseReader)
+	return r
+}
+
+func releaseReader(r *readerWrapper) {
+	r.Release()
+}
+
+type putReaderToPoolOnClose struct {
+	*readerWrapper
+}
+
+func (r *putReaderToPoolOnClose) Close() error {
+	readerPool.Put(r.readerWrapper)
+	return nil
+}
+
+// -- Writer pool
+var writerPool = &sync.Pool{
+	New: newWriter,
+}
+
+type writerWrapper struct {
+	*gozstd.Writer
+}
+
+func newWriter() interface{} {
+	w := &writerWrapper{gozstd.NewWriterLevel(nil, compressionLevel)}
+	runtime.SetFinalizer(w, releaseWriter)
+	return w
+}
+
+func releaseWriter(w *writerWrapper) {
+	w.Release()
+}
+
+type putWriterToPoolOnClose struct {
+	*writerWrapper
+}
+
+func (w putWriterToPoolOnClose) Close() error {
+	err := w.writerWrapper.Close()
+	writerPool.Put(w.writerWrapper)
+	return err
+}
diff --git a/cache/disk/zstdimpl/gozstd.go b/cache/disk/zstdimpl/gozstd.go
new file mode 100644
index 0000000..2b4f0b3
--- /dev/null
+++ b/cache/disk/zstdimpl/gozstd.go
@@ -0,0 +1,68 @@
+package zstdimpl
+
+import (
+	"errors"
+	"github.com/buchgr/bazel-remote/utils/zstdpool"
+	"github.com/klauspost/compress/zstd"
+	syncpool "github.com/mostynb/zstdpool-syncpool"
+	"io"
+)
+
+var zstdFastestLevel = zstd.WithEncoderLevel(zstd.SpeedFastest)
+
+var encoder, _ = zstd.NewWriter(nil, zstdFastestLevel) // TODO: raise WithEncoderConcurrency ?
+var decoder, _ = zstd.NewReader(nil)                   // TODO: raise WithDecoderConcurrency ?
+
+var encoderPool = zstdpool.GetEncoderPool()
+var decoderPool = zstdpool.GetDecoderPool()
+
+var errDecoderPoolFail = errors.New("failed to get decoder from pool")
+var errEncoderPoolFail = errors.New("failed to get encoder from pool")
+
+type goZstd struct{}
+
+func init() {
+	register("go", goZstd{})
+}
+
+// zstdEncoderWrapper is a zstdEncoder that embeds an encoder,
+// and on Close returns it to the pool
+type zstdEncoderWrapper struct {
+	*syncpool.EncoderWrapper
+}
+
+func (w *zstdEncoderWrapper) Close() error {
+	err := w.EncoderWrapper.Close()
+	encoderPool.Put(w.EncoderWrapper)
+	return err
+}
+
+func (goZstd) GetDecoder(in io.ReadCloser) (io.ReadCloser, error) {
+	dec, ok := decoderPool.Get().(*syncpool.DecoderWrapper)
+	if !ok {
+		return nil, errDecoderPoolFail
+	}
+	err := dec.Reset(in)
+	if err != nil {
+		decoderPool.Put(dec)
+		return nil, err
+	}
+	return dec.IOReadCloser(), nil
+}
+
+func (goZstd) GetEncoder(out io.WriteCloser) (zstdEncoder, error) {
+	enc, ok := encoderPool.Get().(*syncpool.EncoderWrapper)
+	if !ok {
+		return nil, errEncoderPoolFail
+	}
+	enc.Reset(out)
+	return &zstdEncoderWrapper{enc}, nil
+}
+
+func (goZstd) DecodeAll(in []byte) ([]byte, error) {
+	return decoder.DecodeAll(in, nil)
+}
+
+func (goZstd) EncodeAll(in []byte) []byte {
+	return encoder.EncodeAll(in, nil)
+}
diff --git a/cache/disk/zstdimpl/zstdimpl.go b/cache/disk/zstdimpl/zstdimpl.go
new file mode 100644
index 0000000..75862a8
--- /dev/null
+++ b/cache/disk/zstdimpl/zstdimpl.go
@@ -0,0 +1,35 @@
+package zstdimpl
+
+import (
+	"fmt"
+	"io"
+)
+
+var registry map[string]ZstdImpl
+
+func register(implName string, impl ZstdImpl) {
+	if registry == nil {
+		registry = make(map[string]ZstdImpl)
+	}
+	registry[implName] = impl
+}
+
+func Get(implName string) (ZstdImpl, error) {
+	impl, ok := registry[implName]
+	if !ok {
+		return nil, fmt.Errorf("Unrecognized ZSTD implementation: %s, supported: %s", implName, registry)
+	}
+	return impl, nil
+}
+
+type ZstdImpl interface {
+	GetDecoder(in io.ReadCloser) (io.ReadCloser, error)
+	GetEncoder(out io.WriteCloser) (zstdEncoder, error)
+	DecodeAll(in []byte) ([]byte, error)
+	EncodeAll(in []byte) []byte
+}
+
+type zstdEncoder interface {
+	io.WriteCloser
+	io.ReaderFrom
+}
diff --git a/cache/httpproxy/BUILD.bazel b/cache/httpproxy/BUILD.bazel
index 6d8c89b..9e39c3a 100644
--- a/cache/httpproxy/BUILD.bazel
+++ b/cache/httpproxy/BUILD.bazel
@@ -8,6 +8,7 @@ go_library(
     deps = [
         "//cache:go_default_library",
         "//cache/disk/casblob:go_default_library",
+        "//cache/disk/zstdimpl:go_default_library",
         "@com_github_prometheus_client_golang//prometheus:go_default_library",
         "@com_github_prometheus_client_golang//prometheus/promauto:go_default_library",
     ],
diff --git a/config/config.go b/config/config.go
index 6b140f3..11675ab 100644
--- a/config/config.go
+++ b/config/config.go
@@ -42,6 +42,7 @@ type Config struct {
 	Dir                         string                    `yaml:"dir"`
 	MaxSize                     int                       `yaml:"max_size"`
 	StorageMode                 string                    `yaml:"storage_mode"`
+	ZstdImplementation          string                    `yaml:"zstd_implementation"`
 	HtpasswdFile                string                    `yaml:"htpasswd_file"`
 	TLSCaFile                   string                    `yaml:"tls_ca_file"`
 	TLSCertFile                 string                    `yaml:"tls_cert_file"`
@@ -93,7 +94,7 @@ var defaultDurationBuckets = []float64{.5, 1, 2.5, 5, 10, 20, 40, 80, 160, 320}
 
 // newFromArgs returns a validated Config with the specified values, and
 // an error if there were any problems with the validation.
-func newFromArgs(dir string, maxSize int, storageMode string,
+func newFromArgs(dir string, maxSize int, storageMode string, zstdImplementation string,
 	httpAddress string, grpcAddress string,
 	profileAddress string,
 	htpasswdFile string,
@@ -126,6 +127,7 @@ func newFromArgs(dir string, maxSize int, storageMode string,
 		Dir:                         dir,
 		MaxSize:                     maxSize,
 		StorageMode:                 storageMode,
+		ZstdImplementation:          zstdImplementation,
 		HtpasswdFile:                htpasswdFile,
 		MaxQueuedUploads:            maxQueuedUploads,
 		NumUploaders:                numUploaders,
@@ -180,6 +182,7 @@ func newFromYaml(data []byte) (*Config, error) {
 	yc := YamlConfig{
 		Config: Config{
 			StorageMode:            "zstd",
+			ZstdImplementation:     "go",
 			NumUploaders:           100,
 			MaxQueuedUploads:       1000000,
 			MaxBlobSize:            math.MaxInt64,
@@ -231,6 +234,9 @@ func validateConfig(c *Config) error {
 	if c.StorageMode != "zstd" && c.StorageMode != "uncompressed" {
 		return errors.New("storage_mode must be set to either \"zstd\" or \"uncompressed\"")
 	}
+	if c.ZstdImplementation != "go" && c.ZstdImplementation != "cgo" {
+		return errors.New("zstd_implementation must be set to either \"go\" or \"cgo\", got: " + c.ZstdImplementation)
+	}
 
 	proxyCount := 0
 	if c.S3CloudStorage != nil {
@@ -480,6 +486,7 @@ func get(ctx *cli.Context) (*Config, error) {
 		ctx.String("dir"),
 		ctx.Int("max_size"),
 		ctx.String("storage_mode"),
+		ctx.String("zstd_implementation"),
 		httpAddress,
 		grpcAddress,
 		profileAddress,
diff --git a/darwin-build.sh b/darwin-build.sh
new file mode 100755
index 0000000..80ec371
--- /dev/null
+++ b/darwin-build.sh
@@ -0,0 +1,10 @@
+#!/bin/bash
+
+set -euxo pipefail
+
+GOARCH=${GOARCH:-arm64}
+
+VERSION_TAG="$(git rev-parse HEAD)"
+VERSION_LINK_FLAG="main.gitCommit=${VERSION_TAG}"
+
+CGO_ENABLED=1 GOOS=darwin GOARCH=$GOARCH go build -a -ldflags "--X ${VERSION_LINK_FLAG}" .
diff --git a/deps.bzl b/deps.bzl
index 1278aa2..7276acc 100644
--- a/deps.bzl
+++ b/deps.bzl
@@ -844,6 +844,16 @@ def go_dependencies():
         sum = "h1:TVEnxayobAdVkhQfrfes2IzOB6o+z4roRkPF52WA1u4=",
         version = "v1.2.1",
     )
+
+    go_repository(
+        name = "com_github_valyala_gozstd",
+        build_file_generation = "off",  # keep
+        importpath = "github.com/valyala/gozstd",
+        patch_args = ["-p1"],  # keep
+        patches = ["gozstd.patch"],  # keep
+        sum = "h1:M4Ds4MIrw+pD+s6vYtuFZ8D3iEw9htzfdytOV3C3iQU=",
+        version = "v1.17.0",
+    )
     go_repository(
         name = "com_github_yuin_goldmark",
         importpath = "github.com/yuin/goldmark",
diff --git a/external/gozstd.patch b/external/gozstd.patch
new file mode 100644
index 0000000..b59ddd7
--- /dev/null
+++ b/external/gozstd.patch
@@ -0,0 +1,142 @@
+From 88b6abf2c7151f9fd8e4bd401fbdf1acb32b4dcb Mon Sep 17 00:00:00 2001
+From: Grzegorz Lukasik <glukasik@nuro.ai>
+Date: Mon, 11 Oct 2021 15:05:52 -0700
+Subject: [PATCH] Added BUILD file that works for bazel.
+
+https://github.com/bazelbuild/bazel-gazelle currently (v0.24.0)
+does not create working go_repository for https://github.com/valyala/gozstd
+
+gazelle creates a go_library, that adds a number of linkopts from
+parsing go files for different platforms, ending with entry like:
+
+  go_library(
+    name = "gozstd",
+    ...
+    clinkopts = select({
+        "@io_bazel_rules_go//go/platform:android_amd64": [
+            "./libzstd_linux_amd64.a",
+        ],
+        "@io_bazel_rules_go//go/platform:android_arm": [
+            "./libzstd_linux_arm.a",
+        ],
+    ...
+
+But the .a files are not included as input files, because of that bazel
+fails during linking with:
+
+  gcc: error: ./libzstd_linux_amd64.a: No such file or directory
+
+This patch commits statically fixed BUILD.bazel file with using
+cc_library that the gozstd library depends on, which causes proper
+linking flags and addition of .a files to the input files.
+---
+ BUILD.bazel | 97 +++++++++++++++++++++++++++++++++++++++++++++++++++++
+ 1 file changed, 97 insertions(+)
+ create mode 100644 BUILD.bazel
+
+diff --git a/BUILD.bazel b/BUILD.bazel
+new file mode 100644
+index 0000000..ce0ff97
+--- /dev/null
++++ b/BUILD.bazel
+@@ -0,0 +1,97 @@
++load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test")
++
++cc_library(
++    name = "gozstd-c-archive",
++    srcs = select({
++        "@io_bazel_rules_go//go/platform:android_amd64": [
++            "libzstd_linux_amd64.a",
++        ],
++        "@io_bazel_rules_go//go/platform:android_arm": [
++            "libzstd_linux_arm.a",
++        ],
++        "@io_bazel_rules_go//go/platform:android_arm64": [
++            "libzstd_linux_arm64.a",
++        ],
++        "@io_bazel_rules_go//go/platform:darwin_amd64": [
++            "libzstd_darwin_amd64.a",
++        ],
++        "@io_bazel_rules_go//go/platform:darwin_arm64": [
++            "libzstd_darwin_arm64.a",
++        ],
++        "@io_bazel_rules_go//go/platform:freebsd_amd64": [
++            "libzstd_freebsd_amd64.a",
++        ],
++        "@io_bazel_rules_go//go/platform:ios_amd64": [
++            "libzstd_darwin_amd64.a",
++        ],
++        "@io_bazel_rules_go//go/platform:ios_arm64": [
++            "libzstd_darwin_arm64.a",
++        ],
++        "@io_bazel_rules_go//go/platform:linux_amd64": [
++            "libzstd_linux_amd64.a",
++        ],
++        "@io_bazel_rules_go//go/platform:linux_arm": [
++            "libzstd_linux_arm.a",
++        ],
++        "@io_bazel_rules_go//go/platform:linux_arm64": [
++            "libzstd_linux_arm64.a",
++        ],
++        "@io_bazel_rules_go//go/platform:windows_amd64": [
++            "libzstd_windows_amd64.a",
++        ]},
++        no_match_error = "Platform not supported by cgo zstd implementation"),
++
++)
++
++go_library(
++    name = "gozstd",
++    srcs = [
++        "dict.go",
++        "doc.go",
++        "gozstd.go",
++        "libzstd_darwin_amd64.go",
++        "libzstd_darwin_arm64.go",
++        "libzstd_freebsd_amd64.go",
++        "libzstd_linux_amd64.go",
++        "libzstd_linux_arm.go",
++        "libzstd_linux_arm64.go",
++        "libzstd_windows_amd64.go",
++        "reader.go",
++        "stream.go",
++        "writer.go",
++        "zdict.h",
++        "zstd.h",
++        "zstd_errors.h",
++    ],
++    cgo = True,
++    cdeps = [":gozstd-c-archive"],
++    copts = ["-O3"],
++    importpath = "github.com/valyala/gozstd",
++    visibility = ["//visibility:public"],
++)
++
++alias(
++    name = "go_default_library",
++    actual = ":gozstd",
++    visibility = ["//visibility:public"],
++)
++
++go_test(
++    name = "gozstd_test",
++    srcs = [
++        "dict_example_test.go",
++        "dict_test.go",
++        "gozstd_example_test.go",
++        "gozstd_test.go",
++        "gozstd_timing_test.go",
++        "reader_example_test.go",
++        "reader_test.go",
++        "reader_timing_test.go",
++        "stream_test.go",
++        "stream_timing_test.go",
++        "writer_example_test.go",
++        "writer_test.go",
++        "writer_timing_test.go",
++    ],
++    embed = [":gozstd"],
++)
+-- 
+2.33.0
+
diff --git a/go.mod b/go.mod
index fab51f7..91aaaec 100644
--- a/go.mod
+++ b/go.mod
@@ -27,12 +27,13 @@ require (
 )
 
 require (
+	github.com/Azure/azure-sdk-for-go/sdk/azcore v1.0.0
 	github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.1.0
 	github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v0.4.1
+	github.com/valyala/gozstd v1.17.0
 )
 
 require (
-	github.com/Azure/azure-sdk-for-go/sdk/azcore v1.0.0 // indirect
 	github.com/Azure/azure-sdk-for-go/sdk/internal v1.0.0 // indirect
 	github.com/AzureAD/microsoft-authentication-library-for-go v0.5.1 // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
diff --git a/go.sum b/go.sum
index 8f6bc30..9ad9318 100644
--- a/go.sum
+++ b/go.sum
@@ -330,6 +330,8 @@ github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/
 github.com/stretchr/testify v1.7.1 h1:5TQK59W5E3v0r2duFAb7P95B6hEeOyEnHRa8MjYSMTY=
 github.com/urfave/cli/v2 v2.3.0 h1:qph92Y649prgesehzOrQjdWyxFOp/QVM+6imKHad91M=
 github.com/urfave/cli/v2 v2.3.0/go.mod h1:LJmUH05zAU44vOAcrfzZQKsZbVcdbOG8rtL3/XcUArI=
+github.com/valyala/gozstd v1.17.0 h1:M4Ds4MIrw+pD+s6vYtuFZ8D3iEw9htzfdytOV3C3iQU=
+github.com/valyala/gozstd v1.17.0/go.mod h1:y5Ew47GLlP37EkTB+B4s7r6A5rdaeB7ftbl9zoYiIPQ=
 github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
diff --git a/linux-build.sh b/linux-build.sh
index 53e322b..1a44956 100755
--- a/linux-build.sh
+++ b/linux-build.sh
@@ -7,4 +7,4 @@ GOARCH=${GOARCH:-amd64}
 VERSION_TAG="$(git rev-parse HEAD)"
 VERSION_LINK_FLAG="main.gitCommit=${VERSION_TAG}"
 
-CGO_ENABLED=0 GOOS=linux GOARCH=$GOARCH go build -a -ldflags "-extldflags '-static' -X ${VERSION_LINK_FLAG}" .
+CGO_ENABLED=1 GOOS=linux GOARCH=$GOARCH go build -a -ldflags "-X ${VERSION_LINK_FLAG}" .
diff --git a/main.go b/main.go
index bf1fa61..bc30f56 100644
--- a/main.go
+++ b/main.go
@@ -87,8 +87,14 @@ func run(ctx *cli.Context) error {
 
 	validateAC := !c.DisableHTTPACValidation
 
+	log.Println("Storage mode:", c.StorageMode)
+	if c.StorageMode == "zstd" {
+		log.Println("Zstandard implementation:", c.ZstdImplementation)
+	}
+
 	opts := []disk.Option{
 		disk.WithStorageMode(c.StorageMode),
+		disk.WithZstdImplementation(c.ZstdImplementation),
 		disk.WithMaxBlobSize(c.MaxBlobSize),
 		disk.WithProxyMaxBlobSize(c.MaxProxyBlobSize),
 		disk.WithAccessLogger(c.AccessLogger),
diff --git a/utils/flags/flags.go b/utils/flags/flags.go
index 23560bd..a04562c 100644
--- a/utils/flags/flags.go
+++ b/utils/flags/flags.go
@@ -47,6 +47,12 @@ func GetCliFlags() []cli.Flag {
 			Usage:   "Which format to store CAS blobs in. Must be one of \"zstd\" or \"uncompressed\".",
 			EnvVars: []string{"BAZEL_REMOTE_STORAGE_MODE"},
 		},
+		&cli.StringFlag{
+			Name:    "zstd_implementation",
+			Value:   "go",
+			Usage:   "ZSTD implementation to use. Must be one of \"go\" or \"cgo\".",
+			EnvVars: []string{"BAZEL_REMOTE_ZSTD_IMPLEMENTATION"},
+		},
 		&cli.StringFlag{
 			Name:    "http_address",
 			Usage:   "Address specification for the HTTP server listener, formatted either as [host]:port for TCP or unix://path.sock for Unix domain sockets.",


Test patch:
diff --git a/.bazelci/system-test.sh b/.bazelci/system-test.sh
index f561ad0..19d61cc 100755
--- a/.bazelci/system-test.sh
+++ b/.bazelci/system-test.sh
@@ -14,6 +14,10 @@ HTTP_PORT=8082
 min_acceptable_hit_rate=95
 overall_result=success
 
+EXTRA_FLAGS=""
+EXTRA_FLAGS_DESC=""
+[ -n "$EXTRA_FLAGS" ] && EXTRA_FLAGS_DESC="(with $EXTRA_FLAGS)"
+
 summary=""
 
 ### Begin minio setup.
@@ -73,10 +77,10 @@ echo "${duration}s"
 # Copy the binary somewhere known, so we can run it manually.
 bazel run --run_under "cp -f " //:bazel-remote $(pwd)/
 
-echo "Starting test cache"
+echo "Starting test cache $EXTRA_FLAGS_DESC"
 test_cache_dir=./bazel-remote-tmp-cache
 rm -rf $test_cache_dir
-./bazel-remote --max_size 1 --dir "$test_cache_dir" --http_address "0.0.0.0:$HTTP_PORT" \
+./bazel-remote --max_size 1 --dir "$test_cache_dir" --http_address "0.0.0.0:$HTTP_PORT" $EXTRA_FLAGS \
 	--s3.endpoint 127.0.0.1:9000 \
 	--s3.bucket bazel-remote \
 	--s3.prefix files \
@@ -103,10 +107,10 @@ grep process http_cold
 
 bazel clean 2> /dev/null
 
-echo "Restarting test cache"
+echo "Restarting test cache $EXTRA_FLAGS_DESC"
 kill -9 $test_cache_pid
 sleep 1
-./bazel-remote --max_size 1 --dir $test_cache_dir --http_address "0.0.0.0:$HTTP_PORT" \
+./bazel-remote --max_size 1 --dir $test_cache_dir --http_address "0.0.0.0:$HTTP_PORT" $EXTRA_FLAGS \
 	> log.stdout 2> log.stderr &
 test_cache_pid=$!
 echo "Test cache pid: $test_cache_pid"
@@ -130,11 +134,11 @@ result=$(awk -vhit_rate=$hit_rate -vmin=$min_acceptable_hit_rate 'BEGIN {if (hit
 echo "hit rate: ${hit_rate}% (hits: $hits misses: $misses) => $result"
 summary+="\n$testsection: hit rate: ${hit_rate}% (hits: $hits misses: $misses) => $result"
 
-echo "Restarting test cache"
+echo "Restarting test cache $EXTRA_FLAGS_DESC"
 kill -9 $test_cache_pid
 sleep 1
 rm -rf $test_cache_dir
-./bazel-remote --max_size 1 --dir $test_cache_dir --http_address "0.0.0.0:$HTTP_PORT" \
+./bazel-remote --max_size 1 --dir $test_cache_dir --http_address "0.0.0.0:$HTTP_PORT" $EXTRA_FLAGS \
 	--s3.endpoint 127.0.0.1:9000 \
 	--s3.bucket bazel-remote \
 	--s3.prefix files \
@@ -168,11 +172,11 @@ result=$(awk -vhit_rate=$hit_rate -vmin=$min_acceptable_hit_rate 'BEGIN {if (hit
 echo "hit rate: ${hit_rate}% (hits: $hits misses: $misses) => $result"
 summary+="\n$testsection: hit rate: ${hit_rate}% (hits: $hits misses: $misses) => $result"
 
-echo "Restarting test cache"
+echo "Restarting test cache $EXTRA_FLAGS_DESC"
 kill -9 $test_cache_pid
 sleep 1
 rm -rf $test_cache_dir
-./bazel-remote --max_size 1 --dir $test_cache_dir --http_address "0.0.0.0:$HTTP_PORT" \
+./bazel-remote --max_size 1 --dir $test_cache_dir --http_address "0.0.0.0:$HTTP_PORT" $EXTRA_FLAGS \
 	> log.stdout 2> log.stderr &
 test_cache_pid=$!
 echo "Test cache pid: $test_cache_pid"
@@ -209,11 +213,11 @@ result=$(awk -vhit_rate=$hit_rate -vmin=$min_acceptable_hit_rate 'BEGIN {if (hit
 echo "hit rate: ${hit_rate}% (hits: $hits misses: $misses) => $result"
 summary+="\n$testsection: hit rate: ${hit_rate}% (hits: $hits misses: $misses) => $result"
 
-echo "Restarting test cache"
+echo "Restarting test cache $EXTRA_FLAGS_DESC"
 kill -9 $test_cache_pid
 sleep 1
 rm -rf $test_cache_dir
-./bazel-remote --max_size 1 --dir $test_cache_dir --http_address "0.0.0.0:$HTTP_PORT" \
+./bazel-remote --max_size 1 --dir $test_cache_dir --http_address "0.0.0.0:$HTTP_PORT" $EXTRA_FLAGS \
 	--s3.endpoint 127.0.0.1:9000 \
 	--s3.bucket bazel-remote \
 	--s3.prefix files \
diff --git a/cache/disk/disk_test.go b/cache/disk/disk_test.go
index 60d5b1f..4c9e6de 100644
--- a/cache/disk/disk_test.go
+++ b/cache/disk/disk_test.go
@@ -21,6 +21,7 @@ import (
 
 	"github.com/buchgr/bazel-remote/cache"
 	"github.com/buchgr/bazel-remote/cache/disk/casblob"
+	"github.com/buchgr/bazel-remote/cache/disk/zstdimpl"
 	"github.com/buchgr/bazel-remote/cache/httpproxy"
 	testutils "github.com/buchgr/bazel-remote/utils"
 
@@ -246,8 +247,16 @@ func (d proxyStub) Get(ctx context.Context, kind cache.EntryKind, hash string) (
 	tfn := tmpfile.Name()
 	defer os.Remove(tfn)
 
-	_, err = casblob.WriteAndClose(ioutil.NopCloser(
-		strings.NewReader(contents)), tmpfile, casblob.Zstandard,
+	var zi zstdimpl.ZstdImpl
+	zi, err = zstdimpl.Get("go")
+	if err != nil {
+		return nil, -1, err
+	}
+
+	_, err = casblob.WriteAndClose(
+		zi,
+		ioutil.NopCloser(
+			strings.NewReader(contents)), tmpfile, casblob.Zstandard,
 		hash, contentsLength)
 	if err != nil {
 		return nil, -1, err
@@ -408,7 +417,12 @@ func TestCacheExistingFiles(t *testing.T) {
 			var f *os.File
 			f, err = os.Create(fp)
 			if err == nil {
-				_, err = casblob.WriteAndClose(r, f, casblob.Zstandard,
+				var zi zstdimpl.ZstdImpl
+				zi, err = zstdimpl.Get("go")
+				if err != nil {
+					t.Fatal(err)
+				}
+				_, err = casblob.WriteAndClose(zi, r, f, casblob.Zstandard,
 					it.hash, int64(len(it.contents)))
 			}
 		} else {
diff --git a/cache/httpproxy/httpproxy_test.go b/cache/httpproxy/httpproxy_test.go
index db864cd..a975211 100644
--- a/cache/httpproxy/httpproxy_test.go
+++ b/cache/httpproxy/httpproxy_test.go
@@ -20,6 +20,7 @@ import (
 	"github.com/buchgr/bazel-remote/cache"
 	"github.com/buchgr/bazel-remote/cache/disk"
 	"github.com/buchgr/bazel-remote/cache/disk/casblob"
+	"github.com/buchgr/bazel-remote/cache/disk/zstdimpl"
 	testutils "github.com/buchgr/bazel-remote/utils"
 )
 
@@ -189,7 +190,14 @@ func TestEverything(t *testing.T) {
 		}
 		defer os.Remove(tmpfile2.Name())
 
-		rc, err := casblob.GetUncompressedReadCloser(tmpfile2, int64(len(casData)), 0)
+		var zi zstdimpl.ZstdImpl
+		zi, err = zstdimpl.Get("go")
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		rc, err := casblob.GetUncompressedReadCloser(
+			zi, tmpfile2, int64(len(casData)), 0)
 		if err != nil {
 			t.Fatal(err)
 		}
diff --git a/config/config_test.go b/config/config_test.go
index 278bd61..63c0515 100644
--- a/config/config_test.go
+++ b/config/config_test.go
@@ -40,6 +40,7 @@ access_log_level: none
 		Dir:                         "/opt/cache-dir",
 		MaxSize:                     100,
 		StorageMode:                 "zstd",
+		ZstdImplementation:          "go",
 		HtpasswdFile:                "/opt/.htpasswd",
 		TLSCertFile:                 "/opt/tls.cert",
 		TLSKeyFile:                  "/opt/tls.key",
@@ -79,11 +80,12 @@ gcs_proxy:
 	}
 
 	expectedConfig := &Config{
-		HTTPAddress: "localhost:8080",
-		GRPCAddress: "localhost:9092",
-		Dir:         "/opt/cache-dir",
-		MaxSize:     100,
-		StorageMode: "zstd",
+		HTTPAddress:        "localhost:8080",
+		GRPCAddress:        "localhost:9092",
+		Dir:                "/opt/cache-dir",
+		MaxSize:            100,
+		StorageMode:        "zstd",
+		ZstdImplementation: "go",
 		GoogleCloudStorage: &GoogleCloudStorageConfig{
 			Bucket:                "gcs-bucket",
 			UseDefaultCredentials: false,
@@ -118,11 +120,12 @@ http_proxy:
 	}
 
 	expectedConfig := &Config{
-		HTTPAddress: "localhost:8080",
-		GRPCAddress: "localhost:9092",
-		Dir:         "/opt/cache-dir",
-		MaxSize:     100,
-		StorageMode: "zstd",
+		HTTPAddress:        "localhost:8080",
+		GRPCAddress:        "localhost:9092",
+		Dir:                "/opt/cache-dir",
+		MaxSize:            100,
+		StorageMode:        "zstd",
+		ZstdImplementation: "go",
 		HTTPBackend: &HTTPBackendConfig{
 			BaseURL: "https://remote-cache.com:8080/cache",
 		},
@@ -188,10 +191,11 @@ s3_proxy:
 	}
 
 	expectedConfig := &Config{
-		HTTPAddress: "localhost:8080",
-		Dir:         "/opt/cache-dir",
-		MaxSize:     100,
-		StorageMode: "zstd",
+		HTTPAddress:        "localhost:8080",
+		Dir:                "/opt/cache-dir",
+		MaxSize:            100,
+		StorageMode:        "zstd",
+		ZstdImplementation: "go",
 		S3CloudStorage: &S3CloudStorageConfig{
 			Endpoint:        "minio.example.com:9000",
 			Bucket:          "test-bucket",
@@ -230,6 +234,7 @@ profile_address: :7070
 		Dir:                    "/opt/cache-dir",
 		MaxSize:                42,
 		StorageMode:            "zstd",
+		ZstdImplementation:     "go",
 		ProfileAddress:         ":7070",
 		NumUploaders:           100,
 		MaxQueuedUploads:       1000000,
@@ -276,6 +281,7 @@ endpoint_metrics_duration_buckets: [.005, .1, 5]
 		Dir:                    "/opt/cache-dir",
 		MaxSize:                42,
 		StorageMode:            "zstd",
+		ZstdImplementation:     "go",
 		NumUploaders:           100,
 		MaxQueuedUploads:       1000000,
 		MaxBlobSize:            math.MaxInt64,
@@ -297,6 +303,7 @@ func TestMetricsDurationBucketsNoDuplicates(t *testing.T) {
 		MaxProxyBlobSize:       math.MaxInt64,
 		Dir:                    "/opt/cache-dir",
 		StorageMode:            "uncompressed",
+		ZstdImplementation:     "go",
 		MetricsDurationBuckets: []float64{1, 2, 3, 3},
 	}
 	err := validateConfig(testConfig)
@@ -370,11 +377,12 @@ storage_mode: gzip
 
 func TestHttpGrpcServerPortConflict(t *testing.T) {
 	testConfig := &Config{
-		HTTPAddress: ":5000",
-		GRPCAddress: ":5000",
-		Dir:         "/opt/cache-dir",
-		MaxSize:     100,
-		StorageMode: "zstd",
+		HTTPAddress:        ":5000",
+		GRPCAddress:        ":5000",
+		Dir:                "/opt/cache-dir",
+		MaxSize:            100,
+		StorageMode:        "zstd",
+		ZstdImplementation: "go",
 	}
 	err := validateConfig(testConfig)
 	if err == nil {
@@ -403,6 +411,7 @@ storage_mode: zstd
 		Dir:                    "/opt/cache-dir",
 		MaxSize:                42,
 		StorageMode:            "zstd",
+		ZstdImplementation:     "go",
 		NumUploaders:           100,
 		MaxQueuedUploads:       1000000,
 		MaxBlobSize:            math.MaxInt64,
@@ -434,6 +443,7 @@ storage_mode: zstd
 		Dir:                    "/opt/cache-dir",
 		MaxSize:                42,
 		StorageMode:            "zstd",
+		ZstdImplementation:     "go",
 		NumUploaders:           100,
 		MaxQueuedUploads:       1000000,
 		MaxBlobSize:            math.MaxInt64,
@@ -449,10 +459,11 @@ storage_mode: zstd
 
 func TestSocketPathMissing(t *testing.T) {
 	testConfig := &Config{
-		HTTPAddress: "unix://",
-		Dir:         "/opt/cache-dir",
-		MaxSize:     100,
-		StorageMode: "zstd",
+		HTTPAddress:        "unix://",
+		Dir:                "/opt/cache-dir",
+		MaxSize:            100,
+		StorageMode:        "zstd",
+		ZstdImplementation: "go",
 	}
 	err := validateConfig(testConfig)
 	if err == nil {