Repository: buchgr/bazel-remote
Pull request #63 (2019-01-04 12:03:01)

Migrate to go modules
Replace `dep` with native go modules for handling 3rd party dependencies. Also update the native (not bazel-based) docker build to go 1.11, as it's required for modules.

These changes don't apply to the Bazel-based dev workflow outlined in the readme, they're just for compatibility with the native golang toolchain.


Patch:
diff --git a/Dockerfile b/Dockerfile
index 4703c62..c941545 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -3,15 +3,10 @@
 
 #
 # Build container
-FROM golang:1.10 AS builder
+FROM golang:1.11 AS builder
 
-# Install dep for vendoring
-ENV GOLANG_DEP_URL https://github.com/golang/dep/releases/download/v0.4.1/dep-linux-amd64
-RUN curl -L -o /usr/local/bin/dep ${GOLANG_DEP_URL} && chmod +x /usr/local/bin/dep
-
-WORKDIR /go/src/github.com/buchgr/bazel-remote
+WORKDIR /src
 COPY . .
-RUN dep ensure
 RUN ./linux-build.sh
 
 #
@@ -19,5 +14,5 @@ RUN ./linux-build.sh
 FROM alpine:latest
 WORKDIR /root
 EXPOSE 80
-COPY --from=0 /go/src/github.com/buchgr/bazel-remote/bazel-remote .
+COPY --from=0 /src/bazel-remote .
 ENTRYPOINT ["./bazel-remote", "--port=80", "--dir=/data"]
diff --git a/Gopkg.lock b/Gopkg.lock
deleted file mode 100644
index d1244c2..0000000
--- a/Gopkg.lock
+++ /dev/null
@@ -1,103 +0,0 @@
-# This file is autogenerated, do not edit; changes may be undone by the next 'dep ensure'.
-
-
-[[projects]]
-  name = "cloud.google.com/go"
-  packages = ["compute/metadata"]
-  revision = "0fd7230b2a7505833d5f69b75cbd6c9582401479"
-  version = "v0.23.0"
-
-[[projects]]
-  name = "github.com/abbot/go-http-auth"
-  packages = ["."]
-  revision = "0ddd408d5d60ea76e320503cc7dd091992dee608"
-  version = "v0.4.0"
-
-[[projects]]
-  name = "github.com/djherbis/atime"
-  packages = ["."]
-  revision = "8e47e0e01d08df8b9f840d74299c8ab70a024a30"
-  version = "v1.0.0"
-
-[[projects]]
-  name = "github.com/golang/protobuf"
-  packages = ["proto"]
-  revision = "b4deda0973fb4c70b50d226b1af49f3da59f5265"
-  version = "v1.1.0"
-
-[[projects]]
-  name = "github.com/google/go-cmp"
-  packages = [
-    "cmp",
-    "cmp/internal/diff",
-    "cmp/internal/function",
-    "cmp/internal/value"
-  ]
-  revision = "3af367b6b30c263d47e8895973edcca9a49cf029"
-  version = "v0.2.0"
-
-[[projects]]
-  name = "github.com/urfave/cli"
-  packages = ["."]
-  revision = "cfb38830724cc34fedffe9a2a29fb54fa9169cd1"
-  version = "v1.20.0"
-
-[[projects]]
-  branch = "master"
-  name = "golang.org/x/crypto"
-  packages = [
-    "bcrypt",
-    "blowfish"
-  ]
-  revision = "ab813273cd59e1333f7ae7bff5d027d4aadf528c"
-
-[[projects]]
-  branch = "master"
-  name = "golang.org/x/net"
-  packages = [
-    "context",
-    "context/ctxhttp"
-  ]
-  revision = "1e491301e022f8f977054da4c2d852decd59571f"
-
-[[projects]]
-  branch = "master"
-  name = "golang.org/x/oauth2"
-  packages = [
-    ".",
-    "google",
-    "internal",
-    "jws",
-    "jwt"
-  ]
-  revision = "ec22f46f877b4505e0117eeaab541714644fdd28"
-
-[[projects]]
-  name = "google.golang.org/appengine"
-  packages = [
-    ".",
-    "internal",
-    "internal/app_identity",
-    "internal/base",
-    "internal/datastore",
-    "internal/log",
-    "internal/modules",
-    "internal/remote_api",
-    "internal/urlfetch",
-    "urlfetch"
-  ]
-  revision = "150dc57a1b433e64154302bdc40b6bb8aefa313a"
-  version = "v1.0.0"
-
-[[projects]]
-  name = "gopkg.in/yaml.v2"
-  packages = ["."]
-  revision = "5420a8b6744d3b0345ab293f6fcba19c978f1183"
-  version = "v2.2.1"
-
-[solve-meta]
-  analyzer-name = "dep"
-  analyzer-version = 1
-  inputs-digest = "3d4dba210844ea80da3cbfa7863333a2692648f1a7dc4ab92be2e3ce86d05464"
-  solver-name = "gps-cdcl"
-  solver-version = 1
diff --git a/Gopkg.toml b/Gopkg.toml
deleted file mode 100644
index 91f37d5..0000000
--- a/Gopkg.toml
+++ /dev/null
@@ -1,15 +0,0 @@
-[[constraint]]
-  name = "github.com/abbot/go-http-auth"
-  version = "0.4.0"
-
-[[constraint]]
-  name = "github.com/djherbis/atime"
-  version = "1.0.0"
-
-[[constraint]]
-  name = "github.com/urfave/cli"
-  version = "1.20.0"
-
-[prune]
-  go-tests = true
-  unused-packages = true
diff --git a/cache/cache.go b/cache/cache.go
index e3cb88c..147674c 100644
--- a/cache/cache.go
+++ b/cache/cache.go
@@ -4,6 +4,23 @@ import (
 	"io"
 )
 
+// EntryKind describes the kind of cache entry
+type EntryKind int
+
+const (
+	// AC stands for Action Cache
+	AC EntryKind = iota
+	// CAS stands for Content Addressable Storage
+	CAS
+)
+
+func (e EntryKind) String() string {
+	if e == AC {
+		return "ac"
+	}
+	return "cas"
+}
+
 // Logger is designed to be satisfied by log.Logger.
 type Logger interface {
 	Printf(format string, v ...interface{})
@@ -27,14 +44,14 @@ type Cache interface {
 
 	// Put stores a stream of `size` bytes from `r` into the cache. If `expectedSha256` is
 	// not the empty string, and the contents don't match it, an error is returned
-	Put(key string, size int64, expectedSha256 string, r io.Reader) error
+	Put(kind EntryKind, hash string, size int64, r io.Reader) error
 
 	// Get writes the content of the cache item stored under `key` to `w`. If the item is
 	// not found, it returns ok = false.
-	Get(key string, actionCache bool) (data io.ReadCloser, sizeBytes int64, err error)
+	Get(kind EntryKind, hash string) (data io.ReadCloser, sizeBytes int64, err error)
 
 	// Contains returns true if the `key` exists.
-	Contains(key string, actionCache bool) (ok bool)
+	Contains(kind EntryKind, hash string) (ok bool)
 
 	// MaxSize returns the maximum cache size in bytes.
 	MaxSize() int64
diff --git a/cache/disk/BUILD.bazel b/cache/disk/BUILD.bazel
index 4c322eb..fa7ee0e 100644
--- a/cache/disk/BUILD.bazel
+++ b/cache/disk/BUILD.bazel
@@ -21,5 +21,8 @@ go_test(
         "lru_test.go",
     ],
     embed = [":go_default_library"],
-    deps = ["//cache:go_default_library"],
+    deps = [
+        "//cache:go_default_library",
+        "//utils:go_default_library",
+    ],
 )
diff --git a/cache/disk/disk.go b/cache/disk/disk.go
index ed8e9bd..c99dd4c 100644
--- a/cache/disk/disk.go
+++ b/cache/disk/disk.go
@@ -38,16 +38,27 @@ type diskCache struct {
 // New returns a new instance of a filesystem-based cache rooted at `dir`,
 // with a maximum size of `maxSizeBytes` bytes.
 func New(dir string, maxSizeBytes int64) cache.Cache {
-	// Create the directory structure
-	ensureDirExists(filepath.Join(dir, "cas"))
-	ensureDirExists(filepath.Join(dir, "ac"))
+	// Create the directory structure.
+	hexLetters := []byte("0123456789abcdef")
+	for _, c1 := range hexLetters {
+		for _, c2 := range hexLetters {
+			subDir := string(c1) + string(c2)
+			err := os.MkdirAll(filepath.Join(dir, cache.CAS.String(), subDir), os.FileMode(0744))
+			if err != nil {
+				log.Fatal(err)
+			}
+			err = os.MkdirAll(filepath.Join(dir, cache.AC.String(), subDir), os.FileMode(0744))
+			if err != nil {
+				log.Fatal(err)
+			}
+		}
+	}
 
 	// The eviction callback deletes the file from disk.
 	onEvict := func(key Key, value SizedItem) {
 		// Only remove committed items (as temporary files have a different filename)
 		if value.(*lruItem).committed {
-			blobPath := filepath.Join(dir, key.(string))
-			err := os.Remove(blobPath)
+			err := os.Remove(filepath.Join(dir, key.(string)))
 			if err != nil {
 				log.Println(err)
 			}
@@ -60,31 +71,64 @@ func New(dir string, maxSizeBytes int64) cache.Cache {
 		lru: NewSizedLRU(maxSizeBytes, onEvict),
 	}
 
-	cache.loadExistingFiles()
+	err := cache.migrateDirectories()
+	if err != nil {
+		log.Fatalf("Attempting to migrate the old directory structure to the new structure failed "+
+			"with error: %v", err)
+	}
+	err = cache.loadExistingFiles()
+	if err != nil {
+		log.Fatalf("Loading of existing cache entries failed due to error: %v", err)
+	}
 
 	return cache
 }
 
-func (c *diskCache) pathForKey(key string) string {
-	return filepath.Join(c.dir, key)
+func (c *diskCache) migrateDirectories() error {
+	err := migrateDirectory(filepath.Join(c.dir, cache.AC.String()))
+	if err != nil {
+		return err
+	}
+	err = migrateDirectory(filepath.Join(c.dir, cache.CAS.String()))
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+func migrateDirectory(dir string) error {
+	return filepath.Walk(dir, func(name string, info os.FileInfo, err error) error {
+		if info.IsDir() {
+			if name == dir {
+				return nil
+			}
+			return filepath.SkipDir
+		}
+		hash := filepath.Base(name)
+		newName := filepath.Join(filepath.Dir(name), hash[:2], hash)
+		return os.Rename(name, newName)
+	})
 }
 
 // loadExistingFiles lists all files in the cache directory, and adds them to the
 // LRU index so that they can be served. Files are sorted by access time first,
 // so that the eviction behavior is preserved across server restarts.
-func (c *diskCache) loadExistingFiles() {
+func (c *diskCache) loadExistingFiles() error {
 	// Walk the directory tree
 	type NameAndInfo struct {
 		info os.FileInfo
 		name string
 	}
 	var files []NameAndInfo
-	filepath.Walk(c.dir, func(name string, info os.FileInfo, err error) error {
+	err := filepath.Walk(c.dir, func(name string, info os.FileInfo, err error) error {
 		if !info.IsDir() {
 			files = append(files, NameAndInfo{info, name})
 		}
 		return nil
 	})
+	if err != nil {
+		return err
+	}
 
 	// Sort in increasing order of atime
 	sort.Slice(files, func(i int, j int) bool {
@@ -92,17 +136,20 @@ func (c *diskCache) loadExistingFiles() {
 	})
 
 	for _, f := range files {
-		key := f.name[len(c.dir)+1:]
-		c.lru.Add(key, &lruItem{
+		relPath := f.name[len(c.dir)+1:]
+		c.lru.Add(relPath, &lruItem{
 			size:      f.info.Size(),
 			committed: true,
 		})
 	}
+	return nil
 }
 
-func (c *diskCache) Put(key string, size int64, expectedSha256 string, r io.Reader) (err error) {
+func (c *diskCache) Put(kind cache.EntryKind, hash string, size int64, r io.Reader) (err error) {
 	c.mux.Lock()
 
+	key := cacheKey(kind, hash)
+
 	// If there's an ongoing upload (i.e. cache key is present in uncommitted state),
 	// we drop the upload and discard the incoming stream. We do accept uploads
 	// of existing keys, as it should happen relatively rarely (e.g. race
@@ -151,15 +198,15 @@ func (c *diskCache) Put(key string, size int64, expectedSha256 string, r io.Read
 	}
 	defer os.Remove(f.Name())
 
-	if expectedSha256 != "" {
+	if kind == cache.CAS {
 		hasher := sha256.New()
 		if _, err = io.Copy(io.MultiWriter(f, hasher), r); err != nil {
 			return
 		}
 		actualHash := hex.EncodeToString(hasher.Sum(nil))
-		if actualHash != expectedSha256 {
+		if actualHash != hash {
 			err = fmt.Errorf(
-				"hashsums don't match. Expected %s, found %s", expectedSha256, actualHash)
+				"hashsums don't match. Expected %s, found %s", key, actualHash)
 			return
 		}
 	} else {
@@ -177,8 +224,8 @@ func (c *diskCache) Put(key string, size int64, expectedSha256 string, r io.Read
 	}
 
 	// Rename to the final path
-	blobPath := c.pathForKey(key)
-	err = os.Rename(f.Name(), blobPath)
+	filePath := cacheFilePath(kind, c.dir, hash)
+	err = os.Rename(f.Name(), filePath)
 	// Only commit if renaming succeeded
 	if err == nil {
 		// This flag is used by the defer() block above.
@@ -188,12 +235,12 @@ func (c *diskCache) Put(key string, size int64, expectedSha256 string, r io.Read
 	return
 }
 
-func (c *diskCache) Get(key string, actionCache bool) (data io.ReadCloser, sizeBytes int64, err error) {
-	if !c.Contains(key, actionCache) {
+func (c *diskCache) Get(kind cache.EntryKind, hash string) (data io.ReadCloser, sizeBytes int64, err error) {
+	if !c.Contains(kind, hash) {
 		return
 	}
 
-	blobPath := c.pathForKey(key)
+	blobPath := cacheFilePath(kind, c.dir, hash)
 
 	fileInfo, err := os.Stat(blobPath)
 	if err != nil {
@@ -209,11 +256,11 @@ func (c *diskCache) Get(key string, actionCache bool) (data io.ReadCloser, sizeB
 	return
 }
 
-func (c *diskCache) Contains(key string, actionCache bool) (ok bool) {
+func (c *diskCache) Contains(kind cache.EntryKind, hash string) (ok bool) {
 	c.mux.Lock()
 	defer c.mux.Unlock()
 
-	val, found := c.lru.Get(key)
+	val, found := c.lru.Get(cacheKey(kind, hash))
 	// Uncommitted (i.e. uploading items) should be reported as not ok
 	return found && val.(*lruItem).committed
 }
@@ -244,3 +291,11 @@ func ensureDirExists(path string) {
 		}
 	}
 }
+
+func cacheKey(kind cache.EntryKind, hash string) string {
+	return filepath.Join(kind.String(), hash[:2], hash)
+}
+
+func cacheFilePath(kind cache.EntryKind, cacheDir string, hash string) string {
+	return filepath.Join(cacheDir, cacheKey(kind, hash))
+}
diff --git a/cache/gcs/gcs.go b/cache/gcs/gcs.go
index 7a2eff1..2fc5bc2 100644
--- a/cache/gcs/gcs.go
+++ b/cache/gcs/gcs.go
@@ -1,5 +1,5 @@
-// A cache that proxies requests to Google Cloud Storage.
-
+// Package gcs provides cache implementation that proxies requests to/from
+// Google Cloud Storage (GCS).
 package gcs
 
 import (
diff --git a/cache/http/http.go b/cache/http/http.go
index d567947..3f1c466 100644
--- a/cache/http/http.go
+++ b/cache/http/http.go
@@ -1,14 +1,15 @@
-// A Cache implementation that can read and write through artifacts to another remote http cache.
+// Package http is a cache implementation that can proxy artifacts from/to another
+// HTTP-based remote cache
 package http
 
 import (
 	"errors"
+	"fmt"
 	"io"
 	"io/ioutil"
 	"net/http"
 	"net/url"
 	"strconv"
-	"strings"
 
 	"github.com/buchgr/bazel-remote/cache"
 )
@@ -17,9 +18,8 @@ const numUploaders = 100
 const maxQueuedUploads = 1000000
 
 type uploadReq struct {
-	key string
-	// true if it's an action cache entry.
-	ac bool
+	hash string
+	kind cache.EntryKind
 }
 
 type remoteHTTPProxyCache struct {
@@ -32,8 +32,8 @@ type remoteHTTPProxyCache struct {
 }
 
 func uploadFile(remote *http.Client, baseURL *url.URL, local cache.Cache, accessLogger cache.Logger,
-	errorLogger cache.Logger, key string, ac bool) {
-	data, size, err := local.Get(key, ac)
+	errorLogger cache.Logger, hash string, kind cache.EntryKind) {
+	data, size, err := local.Get(kind, hash)
 	if err != nil {
 		return
 	}
@@ -42,7 +42,7 @@ func uploadFile(remote *http.Client, baseURL *url.URL, local cache.Cache, access
 		// See https://github.com/golang/go/issues/20257#issuecomment-299509391
 		data = http.NoBody
 	}
-	url := requestURL(baseURL, key, ac)
+	url := requestURL(baseURL, hash, kind)
 	req, err := http.NewRequest(http.MethodPut, url, data)
 	if err != nil {
 		return
@@ -66,7 +66,7 @@ func New(baseURL *url.URL, local cache.Cache, remote *http.Client, accessLogger
 		go func(remote *http.Client, baseURL *url.URL, local cache.Cache, accessLogger cache.Logger,
 			errorLogger cache.Logger) {
 			for item := range uploadQueue {
-				uploadFile(remote, baseURL, local, accessLogger, errorLogger, item.key, item.ac)
+				uploadFile(remote, baseURL, local, accessLogger, errorLogger, item.hash, item.kind)
 			}
 		}(remote, baseURL, local, accessLogger, errorLogger)
 	}
@@ -85,18 +85,17 @@ func logResponse(log cache.Logger, method string, code int, url string) {
 	log.Printf("%4s %d %15s %s", method, code, "", url)
 }
 
-func (r *remoteHTTPProxyCache) Put(key string, size int64, expectedSha256 string, data io.Reader) (err error) {
-	actionCache := expectedSha256 == ""
-	if r.local.Contains(key, actionCache) {
+func (r *remoteHTTPProxyCache) Put(kind cache.EntryKind, hash string, size int64, data io.Reader) (err error) {
+	if r.local.Contains(kind, hash) {
 		io.Copy(ioutil.Discard, data)
 		return nil
 	}
-	r.local.Put(key, size, expectedSha256, data)
+	r.local.Put(kind, hash, size, data)
 
 	select {
 	case r.uploadQueue <- &uploadReq{
-		key: key,
-		ac:  actionCache,
+		hash: hash,
+		kind: kind,
 	}:
 	default:
 		r.errorLogger.Printf("too many uploads queued")
@@ -104,15 +103,15 @@ func (r *remoteHTTPProxyCache) Put(key string, size int64, expectedSha256 string
 	return
 }
 
-func (r *remoteHTTPProxyCache) Get(key string, actionCache bool) (data io.ReadCloser, sizeBytes int64, err error) {
-	if r.local.Contains(key, actionCache) {
-		return r.local.Get(key, actionCache)
+func (r *remoteHTTPProxyCache) Get(kind cache.EntryKind, hash string) (io.ReadCloser, int64, error) {
+	if r.local.Contains(kind, hash) {
+		return r.local.Get(kind, hash)
 	}
 
-	url := requestURL(r.baseURL, key, actionCache)
+	url := requestURL(r.baseURL, hash, kind)
 	rsp, err := r.remote.Get(url)
 	if err != nil {
-		return
+		return nil, -1, err
 	}
 	defer rsp.Body.Close()
 
@@ -136,24 +135,24 @@ func (r *remoteHTTPProxyCache) Get(key string, actionCache bool) (data io.ReadCl
 	sizeBytesStr := rsp.Header.Get("Content-Length")
 	if sizeBytesStr == "" {
 		err = errors.New("Missing Content-Length header")
-		return
+		return nil, -1, err
 	}
 	sizeBytesInt, err := strconv.Atoi(sizeBytesStr)
 	if err != nil {
-		return
+		return nil, -1, err
 	}
-	sizeBytes = int64(sizeBytesInt)
+	sizeBytes := int64(sizeBytesInt)
 
-	err = r.local.Put(key, sizeBytes, "", rsp.Body)
+	err = r.local.Put(kind, hash, sizeBytes, rsp.Body)
 	if err != nil {
-		return
+		return nil, -1, err
 	}
 
-	return r.local.Get(key, actionCache)
+	return r.local.Get(kind, hash)
 }
 
-func (r *remoteHTTPProxyCache) Contains(key string, actionCache bool) (ok bool) {
-	return r.local.Contains(key, actionCache)
+func (r *remoteHTTPProxyCache) Contains(kind cache.EntryKind, hash string) bool {
+	return r.local.Contains(kind, hash)
 }
 
 func (r *remoteHTTPProxyCache) MaxSize() int64 {
@@ -168,11 +167,6 @@ func (r *remoteHTTPProxyCache) NumItems() int {
 	return r.local.NumItems()
 }
 
-func requestURL(baseURL *url.URL, key string, actionCache bool) string {
-	url := baseURL.String()
-	if !strings.HasSuffix(url, "/") {
-		url += "/"
-	}
-	url += key
-	return url
+func requestURL(baseURL *url.URL, hash string, kind cache.EntryKind) string {
+	return fmt.Sprintf("%s/%s/%s", baseURL, kind, hash)
 }
diff --git a/config/config.go b/config/config.go
index a791b4b..70bbad4 100644
--- a/config/config.go
+++ b/config/config.go
@@ -89,8 +89,12 @@ func validateConfig(c *Config) error {
 		return errors.New("The 'dir' flag/key is required")
 	}
 
-	if c.MaxSize == 0 {
-		return errors.New("The 'max_size' flag/key is required")
+	if c.MaxSize <= 0 {
+		return errors.New("The 'max_size' flag/key must be set to a value > 0")
+	}
+
+	if c.Port == 0 {
+		return errors.New("A valid 'port' flag/key must be specified")
 	}
 
 	if (c.TLSCertFile != "" && c.TLSKeyFile == "") || (c.TLSCertFile == "" && c.TLSKeyFile != "") {
diff --git a/go.mod b/go.mod
new file mode 100644
index 0000000..8afb160
--- /dev/null
+++ b/go.mod
@@ -0,0 +1,15 @@
+module github.com/buchgr/bazel-remote
+
+require (
+	cloud.google.com/go v0.23.0
+	github.com/abbot/go-http-auth v0.4.0
+	github.com/djherbis/atime v1.0.0
+	github.com/golang/protobuf v1.1.0
+	github.com/google/go-cmp v0.2.0
+	github.com/urfave/cli v1.20.0
+	golang.org/x/crypto v0.0.0-20180527072434-ab813273cd59
+	golang.org/x/net v0.0.0-20180530234432-1e491301e022
+	golang.org/x/oauth2 v0.0.0-20180529203656-ec22f46f877b
+	google.golang.org/appengine v1.0.0
+	gopkg.in/yaml.v2 v2.2.1
+)
diff --git a/go.sum b/go.sum
new file mode 100644
index 0000000..755166b
--- /dev/null
+++ b/go.sum
@@ -0,0 +1,20 @@
+cloud.google.com/go v0.23.0 h1:w1svupRqvZnfjN9+KksMiggoIRQuMzWkVzpxcR96xDs=
+cloud.google.com/go v0.23.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
+github.com/abbot/go-http-auth v0.4.0 h1:QjmvZ5gSC7jm3Zg54DqWE/T5m1t2AfDu6QlXJT0EVT0=
+github.com/abbot/go-http-auth v0.4.0/go.mod h1:Cz6ARTIzApMJDzh5bRMSUou6UMSp0IEXg9km/ci7TJM=
+github.com/djherbis/atime v1.0.0 h1:ySLvBAM0EvOGaX7TI4dAM5lWj+RdJUCKtGSEHN8SGBg=
+github.com/djherbis/atime v1.0.0/go.mod h1:5W+KBIuTwVGcqjIfaTwt+KSYX1o6uep8dtevevQP/f8=
+github.com/golang/protobuf v1.1.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
+github.com/urfave/cli v1.20.0 h1:fDqGv3UG/4jbVl/QkFwEdddtEDjh/5Ov6X+0B/3bPaw=
+github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA=
+golang.org/x/crypto v0.0.0-20180527072434-ab813273cd59 h1:hk3yo72LXLapY9EXVttc3Z1rLOxT9IuAPPX3GpY2+jo=
+golang.org/x/crypto v0.0.0-20180527072434-ab813273cd59/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
+golang.org/x/net v0.0.0-20180530234432-1e491301e022 h1:MVYFTUmVD3/+ERcvRRI+P/C2+WOUimXh+Pd8LVsklZ4=
+golang.org/x/net v0.0.0-20180530234432-1e491301e022/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/oauth2 v0.0.0-20180529203656-ec22f46f877b h1:nCwwlzLoBQhkY/S3CJ2CGAU4pYfR8+5/TPGEHT+p5Nk=
+golang.org/x/oauth2 v0.0.0-20180529203656-ec22f46f877b/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
+google.golang.org/appengine v1.0.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/yaml.v2 v2.2.1 h1:mUhvW9EsL+naU5Q3cakzfE91YhliOondGd6ZrsDBHQE=
+gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
diff --git a/server/BUILD.bazel b/server/BUILD.bazel
index 7edc2d6..abe9ee6 100644
--- a/server/BUILD.bazel
+++ b/server/BUILD.bazel
@@ -13,6 +13,7 @@ go_test(
     srcs = ["http_test.go"],
     embed = [":go_default_library"],
     deps = [
+        "//cache:go_default_library",
         "//cache/disk:go_default_library",
         "//utils:go_default_library",
     ],
diff --git a/server/http.go b/server/http.go
index 3d318c6..b422462 100644
--- a/server/http.go
+++ b/server/http.go
@@ -7,7 +7,6 @@ import (
 	"io"
 	"net"
 	"net/http"
-	"path/filepath"
 	"regexp"
 	"strconv"
 	"time"
@@ -52,26 +51,27 @@ func NewHTTPCache(cache cache.Cache, accessLogger cache.Logger, errorLogger cach
 }
 
 // Parse cache artifact information from the request URL
-func cacheKeyFromRequestPath(url string) (cacheKey string, sha256sum string, err error) {
+func parseRequestURL(url string) (cache.EntryKind, string, error) {
 	m := blobNameSHA256.FindStringSubmatch(url)
 	if m == nil {
-		err = fmt.Errorf("resource name must be a SHA256 hash in hex. "+
+		err := fmt.Errorf("resource name must be a SHA256 hash in hex. "+
 			"got '%s'", html.EscapeString(url))
-		return
+		return 0, "", err
 	}
 
 	parts := m[2:]
 	if len(parts) != 2 {
-		err = fmt.Errorf("the path '%s' is invalid. expected (ac/|cas/)sha256",
+		err := fmt.Errorf("the path '%s' is invalid. expected (ac/|cas/)sha256",
 			html.EscapeString(url))
-		return
+		return 0, "", err
 	}
 
-	cacheKey = filepath.Join(parts...)
+	// The regex ensures that parts[0] can only be "ac/" or "cas/"
+	hash := parts[1]
 	if parts[0] == "cas/" {
-		sha256sum = parts[1]
+		return cache.CAS, hash, nil
 	}
-	return
+	return cache.AC, hash, nil
 }
 
 func (h *httpCache) CacheHandler(w http.ResponseWriter, r *http.Request) {
@@ -89,22 +89,19 @@ func (h *httpCache) CacheHandler(w http.ResponseWriter, r *http.Request) {
 		h.accessLogger.Printf("%4s %d %15s %s", r.Method, code, clientAddress, r.URL.Path)
 	}
 
-	// Extract cache key from request URL
-	cacheKey, expectedHash, err := cacheKeyFromRequestPath(r.URL.Path)
+	kind, hash, err := parseRequestURL(r.URL.Path)
 	if err != nil {
 		http.Error(w, err.Error(), http.StatusBadRequest)
 		logResponse(http.StatusBadRequest)
 		return
 	}
 
-	fromActionCache := expectedHash == ""
-
 	switch m := r.Method; m {
 	case http.MethodGet:
-		data, sizeBytes, err := h.cache.Get(cacheKey, fromActionCache)
+		data, sizeBytes, err := h.cache.Get(kind, hash)
 		if err != nil {
 			http.Error(w, err.Error(), http.StatusInternalServerError)
-			h.errorLogger.Printf("GET %s: %s", cacheKey, err)
+			h.errorLogger.Printf("GET %s: %s", path(kind, hash), err)
 			return
 		}
 
@@ -123,26 +120,26 @@ func (h *httpCache) CacheHandler(w http.ResponseWriter, r *http.Request) {
 	case http.MethodPut:
 		if r.ContentLength == -1 {
 			// We need the content-length header to make sure we have enough disk space.
-			msg := fmt.Sprintf("PUT without Content-Length (key = %s)", cacheKey)
+			msg := fmt.Sprintf("PUT without Content-Length (key = %s)", path(kind, hash))
 			http.Error(w, msg, http.StatusBadRequest)
 			h.errorLogger.Printf("%s", msg)
 			return
 		}
 
-		err := h.cache.Put(cacheKey, r.ContentLength, expectedHash, r.Body)
+		err := h.cache.Put(kind, hash, r.ContentLength, r.Body)
 		if err != nil {
 			if cerr, ok := err.(*cache.Error); ok {
 				http.Error(w, err.Error(), cerr.Code)
 			} else {
 				http.Error(w, err.Error(), http.StatusInternalServerError)
 			}
-			h.errorLogger.Printf("PUT %s: %s", cacheKey, err)
+			h.errorLogger.Printf("PUT %s: %s", path(kind, hash), err)
 		} else {
 			logResponse(http.StatusOK)
 		}
 
 	case http.MethodHead:
-		ok := h.cache.Contains(cacheKey, fromActionCache)
+		ok := h.cache.Contains(kind, hash)
 		if !ok {
 			http.Error(w, "Not found", http.StatusNotFound)
 			logResponse(http.StatusNotFound)
@@ -172,3 +169,7 @@ func (h *httpCache) StatusPageHandler(w http.ResponseWriter, r *http.Request) {
 		ServerTime: time.Now().Unix(),
 	})
 }
+
+func path(kind cache.EntryKind, hash string) string {
+	return fmt.Sprintf("/%s/%s", kind, hash)
+}


Test patch:
diff --git a/cache/disk/disk_test.go b/cache/disk/disk_test.go
index ca6debd..a36bbd9 100644
--- a/cache/disk/disk_test.go
+++ b/cache/disk/disk_test.go
@@ -2,6 +2,8 @@ package disk
 
 import (
 	"bytes"
+	"crypto/sha256"
+	"encoding/hex"
 	"fmt"
 	"io"
 	"io/ioutil"
@@ -12,6 +14,8 @@ import (
 	"testing"
 	"time"
 
+	"github.com/buchgr/bazel-remote/utils"
+
 	"github.com/buchgr/bazel-remote/cache"
 )
 
@@ -58,12 +62,12 @@ const CONTENTS_HASH = "2cf24dba5fb0a30e26e83b2ac5b9e29e1b161e5c1fa7425e730433629
 func TestCacheBasics(t *testing.T) {
 	cacheDir := tempDir(t)
 	defer os.RemoveAll(cacheDir)
-	cache := New(cacheDir, 100)
+	testCache := New(cacheDir, 100)
 
-	checkItems(t, cache.(*diskCache), 0, 0)
+	checkItems(t, testCache.(*diskCache), 0, 0)
 
 	// Non-existing item
-	data, sizeBytes, err := cache.Get(KEY, false)
+	data, sizeBytes, err := testCache.Get(cache.CAS, CONTENTS_HASH)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -72,17 +76,17 @@ func TestCacheBasics(t *testing.T) {
 	}
 
 	// Add an item
-	err = cache.Put(KEY, int64(len(CONTENTS)), CONTENTS_HASH, strings.NewReader(CONTENTS))
+	err = testCache.Put(cache.CAS, CONTENTS_HASH, int64(len(CONTENTS)), strings.NewReader(CONTENTS))
 	if err != nil {
 		t.Fatal(err)
 	}
 
 	// Dig into the internals to make sure that the cache state has been
 	// updated correctly
-	checkItems(t, cache.(*diskCache), int64(len(CONTENTS)), 1)
+	checkItems(t, testCache.(*diskCache), int64(len(CONTENTS)), 1)
 
 	// Get the item back
-	data, sizeBytes, err = cache.Get(KEY, false)
+	data, sizeBytes, err = testCache.Get(cache.CAS, CONTENTS_HASH)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -92,7 +96,7 @@ func TestCacheBasics(t *testing.T) {
 func TestCacheEviction(t *testing.T) {
 	cacheDir := tempDir(t)
 	defer os.RemoveAll(cacheDir)
-	cache := New(cacheDir, 10)
+	testCache := New(cacheDir, 10)
 
 	expectedSizesNumItems := []struct {
 		expSize int64
@@ -109,12 +113,12 @@ func TestCacheEviction(t *testing.T) {
 	}
 
 	for i, thisExp := range expectedSizesNumItems {
-		err := cache.Put(fmt.Sprintf("key-%d", i), int64(i), "", strings.NewReader("hello"))
+		err := testCache.Put(cache.AC, fmt.Sprintf("aa-%d", i), int64(i), strings.NewReader("hello"))
 		if err != nil {
 			t.Fatal(err)
 		}
 
-		checkItems(t, cache.(*diskCache), thisExp.expSize, thisExp.expNum)
+		checkItems(t, testCache.(*diskCache), thisExp.expSize, thisExp.expNum)
 	}
 }
 
@@ -135,52 +139,51 @@ func expectContentEquals(t *testing.T, data io.ReadCloser, sizeBytes int64, expe
 	}
 }
 
-// Make sure that we can overwrite items if we upload the same key again.
-func TestOverwrite(t *testing.T) {
-	cacheDir := tempDir(t)
-	defer os.RemoveAll(cacheDir)
-	cache := New(cacheDir, 10)
-
-	oldContent := "Hello"
-	newContent := "World"
-
-	err := cache.Put(KEY, 1, "", strings.NewReader(oldContent))
+func putGetCompare(kind cache.EntryKind, hash string, content string, testCache cache.Cache,
+	t *testing.T) {
+	err := testCache.Put(kind, hash, int64(len(content)), strings.NewReader(content))
 	if err != nil {
 		t.Fatal(err)
 	}
 
-	data, sizeBytes, err := cache.Get(KEY, false)
+	data, sizeBytes, err := testCache.Get(kind, hash)
 	if err != nil {
 		t.Fatal(err)
 	}
 	// Get the item back
-	expectContentEquals(t, data, sizeBytes, []byte(oldContent))
+	expectContentEquals(t, data, sizeBytes, []byte(content))
+}
 
-	// Overwrite
-	err = cache.Put(KEY, 1, "", strings.NewReader(newContent))
-	if err != nil {
-		t.Fatal(err)
-	}
+func hashStr(content string) string {
+	hashBytes := sha256.Sum256([]byte(content))
+	return hex.EncodeToString(hashBytes[:])
+}
 
-	data, sizeBytes, err = cache.Get(KEY, false)
-	if err != nil {
-		t.Fatal(err)
-	}
-	// Get the item back again
-	expectContentEquals(t, data, sizeBytes, []byte(newContent))
+// Make sure that we can overwrite items if we upload the same key again.
+func TestOverwrite(t *testing.T) {
+	cacheDir := tempDir(t)
+	defer os.RemoveAll(cacheDir)
+	testCache := New(cacheDir, 10)
+
+	putGetCompare(cache.CAS, hashStr("hello"), "hello", testCache, t)
+	putGetCompare(cache.CAS, hashStr("hello"), "hello", testCache, t)
+
+	putGetCompare(cache.AC, hashStr("world"), "world1", testCache, t)
+	putGetCompare(cache.AC, hashStr("world"), "world2", testCache, t)
 }
 
 func TestCacheExistingFiles(t *testing.T) {
 	cacheDir := tempDir(t)
 	defer os.RemoveAll(cacheDir)
 
-	ensureDirExists(filepath.Join(cacheDir, "cas"))
-	ensureDirExists(filepath.Join(cacheDir, "ac"))
+	ensureDirExists(filepath.Join(cacheDir, "cas", "f5"))
+	ensureDirExists(filepath.Join(cacheDir, "cas", "fd"))
+	ensureDirExists(filepath.Join(cacheDir, "ac", "73"))
 
 	items := []string{
-		"cas/f53b46209596d170f7659a414c9ff9f6b545cf77ffd6e1cbe9bcc57e1afacfbd",
-		"cas/fdce205a735f407ae2910426611893d99ed985e3d9a341820283ea0b7d046ee3",
-		"ac/733e21b37cef883579a88183eed0d00cdeea0b59e1bcd77db6957f881c3a6b54",
+		"cas/f5/f53b46209596d170f7659a414c9ff9f6b545cf77ffd6e1cbe9bcc57e1afacfbd",
+		"cas/fd/fdce205a735f407ae2910426611893d99ed985e3d9a341820283ea0b7d046ee3",
+		"ac/73/733e21b37cef883579a88183eed0d00cdeea0b59e1bcd77db6957f881c3a6b54",
 	}
 
 	for _, it := range items {
@@ -193,17 +196,17 @@ func TestCacheExistingFiles(t *testing.T) {
 	}
 
 	const expectedSize = 3 * int64(len(CONTENTS))
-	cache := New(cacheDir, expectedSize)
+	testCache := New(cacheDir, expectedSize)
 
-	checkItems(t, cache.(*diskCache), expectedSize, 3)
+	checkItems(t, testCache.(*diskCache), expectedSize, 3)
 
 	// Adding a new file should evict items[0] (the oldest)
-	err := cache.Put("a-key", int64(len(CONTENTS)), CONTENTS_HASH, strings.NewReader(CONTENTS))
+	err := testCache.Put(cache.CAS, CONTENTS_HASH, int64(len(CONTENTS)), strings.NewReader(CONTENTS))
 	if err != nil {
 		t.Fatal(err)
 	}
-	checkItems(t, cache.(*diskCache), expectedSize, 3)
-	found := cache.Contains(items[0], false)
+	checkItems(t, testCache.(*diskCache), expectedSize, 3)
+	found := testCache.Contains(cache.CAS, "f53b46209596d170f7659a414c9ff9f6b545cf77ffd6e1cbe9bcc57e1afacfbd")
 	if found {
 		t.Fatalf("%s should have been evicted", items[0])
 	}
@@ -211,12 +214,12 @@ func TestCacheExistingFiles(t *testing.T) {
 
 // Make sure that the cache returns http.StatusInsufficientStorage when trying to upload an item
 // that's bigger than the maximum size.
-func TestCacheBlobtooLarge(t *testing.T) {
+func TestCacheBlobTooLarge(t *testing.T) {
 	cacheDir := tempDir(t)
 	defer os.RemoveAll(cacheDir)
-	diskCache := New(cacheDir, 100)
+	testCache := New(cacheDir, 100)
 
-	err := diskCache.Put("a-key", 10000, "", strings.NewReader(CONTENTS))
+	err := testCache.Put(cache.AC, hashStr("foo"), 10000, strings.NewReader(CONTENTS))
 	if err == nil {
 		t.Fatal("Expected an error")
 	}
@@ -234,10 +237,66 @@ func TestCacheBlobtooLarge(t *testing.T) {
 func TestCacheCorruptedFile(t *testing.T) {
 	cacheDir := tempDir(t)
 	defer os.RemoveAll(cacheDir)
-	cache := New(cacheDir, 1000)
+	testCache := New(cacheDir, 1000)
 
-	err := cache.Put(KEY, int64(len(CONTENTS)), strings.Repeat("x", 64), strings.NewReader(CONTENTS))
+	err := testCache.Put(cache.CAS, hashStr("foo"), int64(len(CONTENTS)), strings.NewReader(CONTENTS))
 	if err == nil {
 		t.Fatal("expected hash mismatch error")
 	}
 }
+
+func TestMigrateFromOldDirectoryStructure(t *testing.T) {
+	cacheDir := testutils.TempDir(t)
+	defer os.RemoveAll(cacheDir)
+
+	acHash, err := testutils.CreateRandomFile(cacheDir+"/ac/", 512)
+	if err != nil {
+		t.Fatal(err)
+	}
+	casHash1, err := testutils.CreateRandomFile(cacheDir+"/cas/", 1024)
+	if err != nil {
+		t.Fatal(err)
+	}
+	casHash2, err := testutils.CreateRandomFile(cacheDir+"/cas/", 1024)
+	if err != nil {
+		t.Fatal(err)
+	}
+	testCache := New(cacheDir, 2560)
+	if testCache.NumItems() != 3 {
+		t.Fatalf("Expected test cache size 3 but was %d", testCache.NumItems())
+	}
+	if !testCache.Contains(cache.AC, acHash) {
+		t.Fatalf("Expected cache to contain AC entry '%s'", acHash)
+	}
+	if !testCache.Contains(cache.CAS, casHash1) {
+		t.Fatalf("Expected cache to contain CAS entry '%s'", casHash1)
+	}
+	if !testCache.Contains(cache.CAS, casHash2) {
+		t.Fatalf("Expected cache to contain CAS entry '%s'", casHash2)
+	}
+}
+
+func TestLoadExistingEntries(t *testing.T) {
+	// Test that loading existing items works
+	cacheDir := testutils.TempDir(t)
+	defer os.RemoveAll(cacheDir)
+	acHash, err := testutils.CreateCacheFile(cacheDir+"/ac/", 1024)
+	if err != nil {
+		t.Fatal(err)
+	}
+	casHash, err := testutils.CreateCacheFile(cacheDir+"/cas/", 1024)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	testCache := New(cacheDir, 2048)
+	if testCache.NumItems() != 2 {
+		t.Fatalf("Expected test cache size 2 but was %d", testCache.NumItems())
+	}
+	if !testCache.Contains(cache.AC, acHash) {
+		t.Fatalf("Expected cache to contain AC entry '%s'", acHash)
+	}
+	if !testCache.Contains(cache.CAS, casHash) {
+		t.Fatalf("Expected cache to contain CAS entry '%s'", casHash)
+	}
+}
diff --git a/cache/http/http_test.go b/cache/http/http_test.go
index 121b18b..7c68c79 100644
--- a/cache/http/http_test.go
+++ b/cache/http/http_test.go
@@ -23,8 +23,6 @@ func TestProxyReadWorks(t *testing.T) {
 	// disk cache.
 
 	expectedData := []byte("hello world")
-	hash := sha256.Sum256(expectedData)
-
 	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 		w.Write(expectedData)
 	}))
@@ -41,13 +39,13 @@ func TestProxyReadWorks(t *testing.T) {
 	proxy := New(baseURL, diskCache, &http.Client{}, testutils.NewSilentLogger(),
 		testutils.NewSilentLogger())
 
-	cacheKey := hex.EncodeToString(hash[:])
-
-	if diskCache.Contains(cacheKey, false) {
+	hashBytes := sha256.Sum256(expectedData)
+	hash := hex.EncodeToString(hashBytes[:])
+	if diskCache.Contains(cache.CAS, hash) {
 		t.Fatalf("Expected the local cache to be empty")
 	}
 
-	readBytes, actualSizeBytes, err := proxy.Get(cacheKey, false)
+	readBytes, actualSizeBytes, err := proxy.Get(cache.CAS, hash)
 	if err != nil {
 		t.Fatalf("Failed to get the blob via the http proxy: '%v'", err)
 	}
@@ -66,7 +64,7 @@ func TestProxyReadWorks(t *testing.T) {
 			len(expectedData))
 	}
 
-	if !diskCache.Contains(cacheKey, false) {
+	if !diskCache.Contains(cache.CAS, hash) {
 		t.Fatalf("Expected the blob to be cached locally.")
 	}
 }
@@ -76,15 +74,15 @@ func TestProxyWriteWorks(t *testing.T) {
 	// disk cache.
 
 	data := []byte("hello world")
-	hash := sha256.Sum256(data)
-	cacheKey := hex.EncodeToString(hash[:])
+	hashBytes := sha256.Sum256(data)
+	hash := hex.EncodeToString(hashBytes[:])
 
 	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 		defer r.Body.Close()
 
-		if !strings.Contains(r.URL.Path, cacheKey) {
+		if !strings.Contains(r.URL.Path, hash) {
 			http.Error(w, fmt.Sprintf("Expected the request URL to contain the key '%s' but was '%s'",
-				cacheKey, r.URL.Path), http.StatusInternalServerError)
+				hash, r.URL.Path), http.StatusInternalServerError)
 			return
 		}
 
@@ -108,17 +106,17 @@ func TestProxyWriteWorks(t *testing.T) {
 	proxy := New(baseURL, diskCache, &http.Client{}, testutils.NewSilentLogger(),
 		testutils.NewSilentLogger())
 
-	if diskCache.Contains(cacheKey, false) {
+	if diskCache.Contains(cache.CAS, hash) {
 		t.Fatalf("Expected the local cache to be empty")
 	}
 
-	err = proxy.Put(cacheKey, int64(len(data)), cacheKey, bytes.NewReader(data))
+	err = proxy.Put(cache.CAS, hash, int64(len(data)), bytes.NewReader(data))
 	if err != nil {
 		t.Errorf("Failed to write to the proxy: '%v'", err)
 	}
 
-	if !diskCache.Contains(cacheKey, false) {
-		t.Fatalf("Expected the local cache to contain '%s'", cacheKey)
+	if !diskCache.Contains(cache.CAS, hash) {
+		t.Fatalf("Expected the local cache to contain '%s'", hash)
 	}
 }
 
@@ -127,7 +125,8 @@ func TestProxyReadErrorsArePropagated(t *testing.T) {
 	// the client.
 
 	expectedData := []byte("hello world")
-	hash := sha256.Sum256(expectedData)
+	hashBytes := sha256.Sum256(expectedData)
+	hash := hex.EncodeToString(hashBytes[:])
 
 	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 		http.Error(w, "Foo bar error", http.StatusForbidden)
@@ -145,8 +144,7 @@ func TestProxyReadErrorsArePropagated(t *testing.T) {
 	proxy := New(baseURL, diskCache, &http.Client{}, testutils.NewSilentLogger(),
 		testutils.NewSilentLogger())
 
-	cacheKey := hex.EncodeToString(hash[:])
-	_, _, err = proxy.Get(cacheKey, false)
+	_, _, err = proxy.Get(cache.CAS, hash)
 	if cerr, ok := err.(*cache.Error); ok {
 		if cerr.Code != http.StatusForbidden {
 			t.Errorf("Expected error code '%d' but got '%d'", http.StatusForbidden, cerr.Code)
@@ -166,7 +164,8 @@ func TestProxyWriteErrorsAreNotPropagated(t *testing.T) {
 	// basis.
 
 	data := []byte("hello world")
-	hash := sha256.Sum256(data)
+	hashBytes := sha256.Sum256(data)
+	hash := hex.EncodeToString(hashBytes[:])
 
 	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 		http.Error(w, "Foo bar error", http.StatusForbidden)
@@ -184,14 +183,12 @@ func TestProxyWriteErrorsAreNotPropagated(t *testing.T) {
 	proxy := New(baseURL, diskCache, &http.Client{}, testutils.NewSilentLogger(),
 		testutils.NewSilentLogger())
 
-	cacheKey := hex.EncodeToString(hash[:])
-
-	err = proxy.Put(cacheKey, int64(len(data)), cacheKey, bytes.NewReader(data))
+	err = proxy.Put(cache.CAS, hash, int64(len(data)), bytes.NewReader(data))
 	if err != nil {
 		t.Error("Expected the error on put to not be propagated")
 	}
 
-	if !diskCache.Contains(cacheKey, false) {
+	if !diskCache.Contains(cache.CAS, hash) {
 		t.Error("Expected the blob to be stored locally")
 	}
 }
diff --git a/config/config_test.go b/config/config_test.go
index dfb1fe3..1371718 100644
--- a/config/config_test.go
+++ b/config/config_test.go
@@ -2,6 +2,7 @@ package config
 
 import (
 	"reflect"
+	"strings"
 	"testing"
 
 	"github.com/google/go-cmp/cmp"
@@ -96,3 +97,33 @@ http_proxy:
 		t.Fatalf("Expected '%+v' but got '%+v'", expectedConfig, config)
 	}
 }
+
+func TestDirRequired(t *testing.T) {
+	testConfig := &Config{
+		Host:    "localhost",
+		Port:    8080,
+		MaxSize: 100,
+	}
+	err := validateConfig(testConfig)
+	if err == nil {
+		t.Fatal("Expected an error because no 'dir' was specified")
+	}
+	if !strings.Contains(err.Error(), "'dir'") {
+		t.Fatal("Expected the error message to mention the missing 'dir' key/flag")
+	}
+}
+
+func TestMaxSizeRequired(t *testing.T) {
+	testConfig := &Config{
+		Host: "localhost",
+		Port: 8080,
+		Dir:  "/opt/cache-dir",
+	}
+	err := validateConfig(testConfig)
+	if err == nil {
+		t.Fatal("Expected an error because no 'max_size' was specified")
+	}
+	if !strings.Contains(err.Error(), "'max_size'") {
+		t.Fatal("Expected the error message to mention the missing 'max_size' key/flag")
+	}
+}
diff --git a/server/http_test.go b/server/http_test.go
index 8d26d28..98151cb 100644
--- a/server/http_test.go
+++ b/server/http_test.go
@@ -14,6 +14,8 @@ import (
 	"sync"
 	"testing"
 
+	"github.com/buchgr/bazel-remote/cache"
+
 	"github.com/buchgr/bazel-remote/cache/disk"
 	"github.com/buchgr/bazel-remote/utils"
 )
@@ -22,7 +24,7 @@ func TestDownloadFile(t *testing.T) {
 	cacheDir := testutils.CreateTmpCacheDirs(t)
 	defer os.RemoveAll(cacheDir)
 
-	hash, err := testutils.CreateRandomFile(filepath.Join(cacheDir, "cas"), 1024)
+	hash, err := testutils.CreateCacheFile(filepath.Join(cacheDir, "cas"), 1024)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -242,9 +244,9 @@ func TestStatusPage(t *testing.T) {
 	}
 }
 
-func TestCacheKeyFromRequestPath(t *testing.T) {
+func TestParseRequestURL(t *testing.T) {
 	{
-		_, _, err := cacheKeyFromRequestPath("invalid/url")
+		_, _, err := parseRequestURL("invalid/url")
 		if err == nil {
 			t.Error("Failed to reject an invalid URL")
 		}
@@ -253,42 +255,41 @@ func TestCacheKeyFromRequestPath(t *testing.T) {
 	const aSha256sum = "fec3be77b8aa0d307ed840581ded3d114c86f36d4914c81e33a72877020c0603"
 
 	{
-		cacheKey, shasum, err := cacheKeyFromRequestPath("cas/" + aSha256sum)
+		kind, hash, err := parseRequestURL("cas/" + aSha256sum)
 		if err != nil {
 			t.Error("Failed to parse a valid CAS URL")
 		}
-		if cacheKey != "cas/"+aSha256sum {
+		if hash != aSha256sum {
 			t.Error("Cache key parsed incorrectly")
 		}
-		if shasum != aSha256sum {
-			t.Log(shasum)
-			t.Error("Hashsum parsed incorrectly")
+		if kind != cache.CAS {
+			t.Errorf("Expected kind CAS but got AC")
 		}
 	}
 
 	{
-		cacheKey, shasum, err := cacheKeyFromRequestPath("ac/" + aSha256sum)
+		kind, hash, err := parseRequestURL("ac/" + aSha256sum)
 		if err != nil {
 			t.Error("Failed to parse a valid AC URL")
 		}
-		if cacheKey != "ac/"+aSha256sum {
+		if hash != aSha256sum {
 			t.Error("Cache key parsed incorrectly")
 		}
-		if shasum != "" {
-			t.Error("Hashsum parsed incorrectly")
+		if kind != cache.AC {
+			t.Error("Expected kind AC but got CAS")
 		}
 	}
 
 	{
-		cacheKey, shasum, err := cacheKeyFromRequestPath("prefix/ac/" + aSha256sum)
+		kind, hash, err := parseRequestURL("prefix/ac/" + aSha256sum)
 		if err != nil {
 			t.Error("Failed to parse a valid AC URL with prefix")
 		}
-		if cacheKey != "ac/"+aSha256sum {
+		if hash != aSha256sum {
 			t.Error("Cache key parsed incorrectly")
 		}
-		if shasum != "" {
-			t.Error("Hashsum parsed incorrectly")
+		if kind != cache.AC {
+			t.Error("Expected kind AC but got CAS")
 		}
 	}
 }
diff --git a/utils/testutils.go b/utils/testutils.go
index 55d48a5..d42a822 100644
--- a/utils/testutils.go
+++ b/utils/testutils.go
@@ -7,7 +7,6 @@ import (
 	"io/ioutil"
 	"log"
 	"os"
-	"path/filepath"
 	"testing"
 )
 
@@ -20,10 +19,20 @@ func TempDir(t *testing.T) string {
 }
 
 func CreateRandomFile(dir string, size int64) (string, error) {
-	data, filename := RandomDataAndHash(size)
-	filepath := dir + "/" + filename
+	data, hash := RandomDataAndHash(size)
+	os.MkdirAll(dir, os.FileMode(0744))
+	filepath := dir + "/" + hash
 
-	return filename, ioutil.WriteFile(filepath, data, 0744)
+	return hash, ioutil.WriteFile(filepath, data, 0744)
+}
+
+func CreateCacheFile(dir string, size int64) (string, error) {
+	data, hash := RandomDataAndHash(size)
+	subdir := dir + "/" + hash[0:2]
+	os.MkdirAll(subdir, os.FileMode(0744))
+	filepath := subdir + "/" + hash
+
+	return hash, ioutil.WriteFile(filepath, data, 0744)
 }
 
 func RandomDataAndHash(size int64) ([]byte, string) {
@@ -39,9 +48,7 @@ func CreateTmpCacheDirs(t *testing.T) string {
 	if err != nil {
 		t.Error("Couldn't create tmp dir", err)
 	}
-	EnsureDirExists(filepath.Join(path, "ac"))
-	EnsureDirExists(filepath.Join(path, "cas"))
-
+	os.MkdirAll(path, os.FileMode(0744))
 	return path
 }
 
@@ -50,12 +57,3 @@ func CreateTmpCacheDirs(t *testing.T) string {
 func NewSilentLogger() *log.Logger {
 	return log.New(ioutil.Discard, "", 0)
 }
-
-func EnsureDirExists(path string) {
-	if _, err := os.Stat(path); os.IsNotExist(err) {
-		err = os.MkdirAll(path, os.FileMode(0744))
-		if err != nil {
-			log.Fatal(err)
-		}
-	}
-}