Repository: buchgr/bazel-remote
Pull request #691 (2023-08-29 19:47:06)

Fix serving uncompressed blob as compressed
The pipe is created but never written to nor closed. Instead, the file is passed as writer to the encoder, causing errors like the following when serving zstd compressed blob when using uncompressed storage:
```
2023/08/29 18:58:44 Error while closing encoder: write /private/tmp/cache/cas.v2/4d/4d42748448b9cf8484bfea1e5df65e33213fcc85bba393ae4f3ebebd627523c0-144750847.v1: bad file descriptor
2023/08/29 18:58:44 Error writing cas/4d42748448b9cf8484bfea1e5df65e33213fcc85bba393ae4f3ebebd627523c0 err: write /private/tmp/cache/cas.v2/4d/4d42748448b9cf8484bfea1e5df65e33213fcc85bba393ae4f3ebebd627523c0-144750847.v1: bad file descriptor
2023/08/29 18:58:44  GET 500       127.0.0.1 /cas/4d42748448b9cf8484bfea1e5df65e33213fcc85bba393ae4f3ebebd627523c0
```


Patch:
diff --git a/BUILD.bazel b/BUILD.bazel
index 3b220f1..1109681 100644
--- a/BUILD.bazel
+++ b/BUILD.bazel
@@ -85,7 +85,7 @@ go_binary(
     goarch = "amd64",
     goos = "darwin",
     pure = "off",
-    static = "on",
+    #static = "on", # With static enabled, I get the following error: "ld: library not found for -lcrt0.o"
     visibility = ["//visibility:public"],
     x_defs = {"main.gitCommit": "{STABLE_GIT_COMMIT}"},
 )
@@ -97,7 +97,7 @@ go_binary(
     goarch = "arm64",
     goos = "darwin",
     pure = "off",
-    static = "on",
+    #static = "on", # With static enabled, I get the following error: "ld: library not found for -lcrt0.o"
     visibility = ["//visibility:public"],
     x_defs = {"main.gitCommit": "{STABLE_GIT_COMMIT}"},
 )
diff --git a/cache/disk/casblob/BUILD.bazel b/cache/disk/casblob/BUILD.bazel
index eb6f88f..64df551 100644
--- a/cache/disk/casblob/BUILD.bazel
+++ b/cache/disk/casblob/BUILD.bazel
@@ -11,4 +11,9 @@ go_library(
 go_test(
     name = "go_default_test",
     srcs = ["casblob_test.go"],
+    deps = [
+        ":go_default_library",
+        "//cache/disk/zstdimpl:go_default_library",
+        "//utils:go_default_library",
+    ],
 )
diff --git a/cache/disk/casblob/casblob.go b/cache/disk/casblob/casblob.go
index 12a5330..4d0d465 100644
--- a/cache/disk/casblob/casblob.go
+++ b/cache/disk/casblob/casblob.go
@@ -402,7 +402,7 @@ func GetLegacyZstdReadCloser(zstd zstdimpl.ZstdImpl, f *os.File) (io.ReadCloser,
 
 	pr, pw := io.Pipe()
 
-	enc, err := zstd.GetEncoder(f)
+	enc, err := zstd.GetEncoder(pw)
 	if err != nil {
 		_ = f.Close()
 		return nil, err
@@ -425,7 +425,14 @@ func GetLegacyZstdReadCloser(zstd zstdimpl.ZstdImpl, f *os.File) (io.ReadCloser,
 			log.Println("Error while closing encoder:", err)
 			_ = pw.CloseWithError(err)
 		}
-		_ = f.Close()
+
+		err = f.Close()
+		if err != nil {
+			log.Println("Error while closing file:", err)
+			_ = pw.CloseWithError(err)
+		}
+
+		_ = pw.Close()
 	}()
 
 	return pr, nil
diff --git a/utils/flags/flags.go b/utils/flags/flags.go
index 22b6de4..9000244 100644
--- a/utils/flags/flags.go
+++ b/utils/flags/flags.go
@@ -40,7 +40,6 @@ func GetCliFlags() []cli.Flag {
 			Name:     "max_size",
 			Usage:    "The maximum size of bazel-remote's disk cache in GiB. This flag is required.",
 			EnvVars:  []string{"BAZEL_REMOTE_MAX_SIZE"},
-			Required: true,
 		},
 		&cli.StringFlag{
 			Name:    "storage_mode",


Test patch:
diff --git a/cache/disk/casblob/casblob_test.go b/cache/disk/casblob/casblob_test.go
index c656005..599d728 100644
--- a/cache/disk/casblob/casblob_test.go
+++ b/cache/disk/casblob/casblob_test.go
@@ -1,8 +1,18 @@
 package casblob_test
 
 import (
+	"bytes"
+	"crypto/sha256"
+	"encoding/hex"
+	"fmt"
+	"io"
+	"os"
 	"testing"
 	"unsafe"
+
+	"github.com/buchgr/bazel-remote/v2/cache/disk/casblob"
+	"github.com/buchgr/bazel-remote/v2/cache/disk/zstdimpl"
+	testutils "github.com/buchgr/bazel-remote/v2/utils"
 )
 
 func TestLenSize(t *testing.T) {
@@ -17,3 +27,55 @@ func TestLenSize(t *testing.T) {
 		t.Errorf("This should silence linters that think slice is never used")
 	}
 }
+
+func TestZstdFromLegacy(t *testing.T) {
+	size := 1024
+	zstd, err := zstdimpl.Get("go")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	data, hash := testutils.RandomDataAndHash(int64(size))
+	dir := testutils.TempDir(t)
+	filename := fmt.Sprintf("%s/%s", dir, hash)
+	file, err := os.OpenFile(filename, os.O_RDWR|os.O_CREATE|os.O_EXCL, 0664)
+	if err != nil {
+		t.Fatal(err)
+	}
+	n, err := file.Write(data)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if n != size {
+		t.Fatalf("Unexpected short write %d, expected %d", n, size)
+	}
+	file.Close()
+
+	file, err = os.Open(filename)
+	if err != nil {
+		t.Fatal(err)
+	}
+	zrc, err := casblob.GetLegacyZstdReadCloser(zstd, file)
+	if err != nil {
+		t.Fatal(err)
+	}
+	rc, err := zstd.GetDecoder(zrc)
+	if err != nil {
+		t.Fatal(err)
+	}
+	buf := bytes.NewBuffer(nil)
+	_, err = io.Copy(buf, rc)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if buf.Len() != size {
+		t.Fatalf("Unexpected buf size %d, expected %d", buf.Len(), size)
+	}
+
+	h := sha256.Sum256(data)
+	hs := hex.EncodeToString(h[:])
+	if hs != hash {
+		t.Fatalf("Unexpected content sha %s, expected %s", hs, hash)
+	}
+}