Repository: buchgr/bazel-remote
Pull request #111 (2019-10-28 08:01:11)

validate ActionCache items before returning them for http also
A common bug is the scenario where an ActionResult exists in the
ActionCache and is returned to the client, but not all the blobs
that it depends on are available in the CAS when the client tries
to get them. The client typically errors out at this point, and
might also continue failing when re-run, until the ActionResult
is removed from the cache.

To avoid this problem, we should verify that all the required
blobs are available before returning a given ActionCache. Doing
so should move these items to the "safe" end of the LRU queue,
and increase the odds that the CAS blobs will still exist in the
cache when the client requests them.

We previously had this check in the gRPC interface, but not for
the REST interface. This change refactors the check so it's
available to the REST interface. Since some clients might be
relying on bazel-remote to store arbitrary non-ActionResult blobs
in the ActionCache via the REST interface, we provide a command
line flag to disable this validation:
--disable_http_ac_validation

When --disable_http_ac_validation is specified, ActionCache blobs
uploaded and downloaded via REST are stored in a new "raw"
keyspace to avoid clashes with validated gRPC ActionResult blobs
(which are always validated and always stored in the "ac" keyspace).

If you want to use --disable_http_ac_validation and not lose access
to existing cache items, stop bazel-remote, rename the "ac" dir to
"raw", then restart bazel-remote with --disable_http_ac_validation.

mostynb: TODO before this is ready:
* Decide if we should validate CAS blob availability on PUT.
* Add the new flag to README.md (https://github.com/buchgr/bazel-remote/pull/110 should land first).
* Decide if we should expose a "raw" HTTP endpoint, which would allow both validated and unvalidated blob uploads via HTTP.
buchgr: > Decide if we should validate CAS blob availability on PUT.

I think having this check on GET is sufficient for now. I think ideally we'd have a GC with reference counting. That is, we'd only evict an output if no action references it or similarly if we evict and output also evict all actions that reference it.  That way we would only need to do the check on upload and not on download and thus also not have any performance cost on GET.
mostynb: Updated the readme, removed a TODO note in the PUT section. I like the idea of having a separate "raw/" REST endpoint, but this version is also OK for my use cases.

Patch:
diff --git a/README.md b/README.md
index 53d03ca..587570e 100644
--- a/README.md
+++ b/README.md
@@ -66,6 +66,7 @@ GLOBAL OPTIONS:
    --s3.access_key_id value      The S3/minio access key to use when using S3 cache backend. [$BAZEL_REMOTE_S3_ACCESS_KEY_ID]
    --s3.secret_access_key value  The S3/minio secret access key to use when using S3 cache backend. [$BAZEL_REMOTE_S3_SECRET_ACCESS_KEY]
    --s3.disable_ssl              Whether to disable TLS/SSL when using the S3 cache backend.  Default is false (enable TLS/SSL). [$BAZEL_REMOTE_S3_DISABLE_SSL]
+   --disable_http_ac_validation  Whether to disable ActionResult validation for HTTP requests.  Default is false (enable validation). [$BAZEL_REMOTE_DISABLE_HTTP_AC_VALIDATION]
    --help, -h                    show help
 ```
 
diff --git a/WORKSPACE b/WORKSPACE
index c970e9c..1dbbda8 100644
--- a/WORKSPACE
+++ b/WORKSPACE
@@ -1,6 +1,5 @@
 load("@bazel_tools//tools/build_defs/repo:git.bzl", "git_repository")
 load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive")
-
 load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive")
 
 http_archive(
@@ -106,8 +105,8 @@ go_repository(
 go_repository(
     # minio has this dependency
     name = "com_github_go_ini_ini",
-    importpath="github.com/go-ini/ini",
-    commit="9c8236e659b76e87bf02044d06fde8683008ff3e",
+    importpath = "github.com/go-ini/ini",
+    commit = "9c8236e659b76e87bf02044d06fde8683008ff3e",
 )
 
 go_repository(
diff --git a/cache/BUILD.bazel b/cache/BUILD.bazel
index 14417d9..e3baa19 100644
--- a/cache/BUILD.bazel
+++ b/cache/BUILD.bazel
@@ -5,4 +5,8 @@ go_library(
     srcs = ["cache.go"],
     importpath = "github.com/buchgr/bazel-remote/cache",
     visibility = ["//visibility:public"],
+    deps = [
+        "@com_github_bazelbuild_remote_apis//build/bazel/remote/execution/v2:go_default_library",
+        "@com_github_golang_protobuf//proto:go_default_library",
+    ],
 )
diff --git a/cache/cache.go b/cache/cache.go
index 4a8fe20..01558fa 100644
--- a/cache/cache.go
+++ b/cache/cache.go
@@ -2,6 +2,10 @@ package cache
 
 import (
 	"io"
+	"io/ioutil"
+
+	pb "github.com/bazelbuild/remote-apis/build/bazel/remote/execution/v2"
+	"github.com/golang/protobuf/proto"
 )
 
 // EntryKind describes the kind of cache entry
@@ -12,13 +16,20 @@ const (
 	AC EntryKind = iota
 	// CAS stands for Content Addressable Storage
 	CAS
+	// Like Action Cache, but without ActionResult validation.
+	// Not exposed externally, only used for HTTP when running with
+	// the --disable_http_ac_validation commandline flag.
+	RAW
 )
 
 func (e EntryKind) String() string {
 	if e == AC {
 		return "ac"
 	}
-	return "cas"
+	if e == CAS {
+		return "cas"
+	}
+	return "raw"
 }
 
 // Logger is designed to be satisfied by log.Logger.
@@ -64,3 +75,57 @@ type Cache interface {
 	// NumItems returns the number of items stored in the cache.
 	NumItems() int
 }
+
+// If `hash` refers to a valid ActionResult with all the dependencies
+// available in the CAS, return it and its serialized value.
+// If not, return nil values.
+// If something unexpected went wrong, return an error.
+func GetValidatedActionResult(c Cache, hash string) (*pb.ActionResult, []byte, error) {
+	rdr, sizeBytes, err := c.Get(AC, hash)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	if rdr == nil || sizeBytes <= 0 {
+		return nil, nil, nil // aka "not found"
+	}
+
+	data, err := ioutil.ReadAll(rdr)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	result := &pb.ActionResult{}
+	err = proto.Unmarshal(data, result)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	for _, f := range result.OutputFiles {
+		if len(f.Contents) == 0 && f.Digest.SizeBytes > 0 {
+			if !c.Contains(CAS, f.Digest.Hash) {
+				return nil, nil, nil // aka "not found"
+			}
+		}
+	}
+
+	for _, d := range result.OutputDirectories {
+		if !c.Contains(CAS, d.TreeDigest.Hash) {
+			return nil, nil, nil // aka "not found"
+		}
+	}
+
+	if result.StdoutDigest != nil {
+		if !c.Contains(CAS, result.StdoutDigest.Hash) {
+			return nil, nil, nil // aka "not found"
+		}
+	}
+
+	if result.StderrDigest != nil {
+		if !c.Contains(CAS, result.StderrDigest.Hash) {
+			return nil, nil, nil // aka "not found"
+		}
+	}
+
+	return result, data, nil
+}
diff --git a/cache/disk/disk.go b/cache/disk/disk.go
index 42f89cc..c4e0feb 100644
--- a/cache/disk/disk.go
+++ b/cache/disk/disk.go
@@ -51,6 +51,10 @@ func New(dir string, maxSizeBytes int64) cache.Cache {
 			if err != nil {
 				log.Fatal(err)
 			}
+			err = os.MkdirAll(filepath.Join(dir, cache.RAW.String(), subDir), os.FileMode(0744))
+			if err != nil {
+				log.Fatal(err)
+			}
 		}
 	}
 
@@ -93,6 +97,7 @@ func (c *diskCache) migrateDirectories() error {
 	if err != nil {
 		return err
 	}
+	// Note: there are no old "RAW" directories (yet).
 	return nil
 }
 
diff --git a/config/config.go b/config/config.go
index 196f8a9..1652162 100644
--- a/config/config.go
+++ b/config/config.go
@@ -31,37 +31,39 @@ type HTTPBackendConfig struct {
 
 // Config provides the configuration
 type Config struct {
-	Host               string                    `yaml:"host"`
-	Port               int                       `yaml:"port"`
-	GRPCPort           int                       `yaml:"grpc_port"`
-	Dir                string                    `yaml:"dir"`
-	MaxSize            int                       `yaml:"max_size"`
-	HtpasswdFile       string                    `yaml:"htpasswd_file"`
-	TLSCertFile        string                    `yaml:"tls_cert_file"`
-	TLSKeyFile         string                    `yaml:"tls_key_file"`
-	S3CloudStorage     *S3CloudStorageConfig     `yaml:"s3_proxy"`
-	GoogleCloudStorage *GoogleCloudStorageConfig `yaml:"gcs_proxy"`
-	HTTPBackend        *HTTPBackendConfig        `yaml:"http_proxy"`
-	IdleTimeout        time.Duration             `yaml:"idle_timeout"`
+	Host                    string                    `yaml:"host"`
+	Port                    int                       `yaml:"port"`
+	GRPCPort                int                       `yaml:"grpc_port"`
+	Dir                     string                    `yaml:"dir"`
+	MaxSize                 int                       `yaml:"max_size"`
+	HtpasswdFile            string                    `yaml:"htpasswd_file"`
+	TLSCertFile             string                    `yaml:"tls_cert_file"`
+	TLSKeyFile              string                    `yaml:"tls_key_file"`
+	S3CloudStorage          *S3CloudStorageConfig     `yaml:"s3_proxy"`
+	GoogleCloudStorage      *GoogleCloudStorageConfig `yaml:"gcs_proxy"`
+	HTTPBackend             *HTTPBackendConfig        `yaml:"http_proxy"`
+	IdleTimeout             time.Duration             `yaml:"idle_timeout"`
+	DisableHTTPACValidation bool                      `yaml:"disable_http_ac_validation"`
 }
 
 // New ...
 func New(dir string, maxSize int, host string, port int, grpc_port int, htpasswdFile string,
 	tlsCertFile string, tlsKeyFile string, idleTimeout time.Duration,
-	s3 *S3CloudStorageConfig) (*Config, error) {
+	s3 *S3CloudStorageConfig, disable_http_ac_validation bool) (*Config, error) {
 	c := Config{
-		Host:               host,
-		Port:               port,
-		GRPCPort:           grpc_port,
-		Dir:                dir,
-		MaxSize:            maxSize,
-		HtpasswdFile:       htpasswdFile,
-		TLSCertFile:        tlsCertFile,
-		TLSKeyFile:         tlsKeyFile,
-		S3CloudStorage:     s3,
-		GoogleCloudStorage: nil,
-		HTTPBackend:        nil,
-		IdleTimeout:        idleTimeout,
+		Host:                    host,
+		Port:                    port,
+		GRPCPort:                grpc_port,
+		Dir:                     dir,
+		MaxSize:                 maxSize,
+		HtpasswdFile:            htpasswdFile,
+		TLSCertFile:             tlsCertFile,
+		TLSKeyFile:              tlsKeyFile,
+		S3CloudStorage:          s3,
+		GoogleCloudStorage:      nil,
+		HTTPBackend:             nil,
+		IdleTimeout:             idleTimeout,
+		DisableHTTPACValidation: disable_http_ac_validation,
 	}
 
 	err := validateConfig(&c)
diff --git a/main.go b/main.go
index 84084b3..f6d1a66 100644
--- a/main.go
+++ b/main.go
@@ -135,6 +135,11 @@ func main() {
 			Usage:  "Whether to disable TLS/SSL when using the S3 cache backend.  Default is false (enable TLS/SSL).",
 			EnvVar: "BAZEL_REMOTE_S3_DISABLE_SSL",
 		},
+		cli.BoolFlag{
+			Name:   "disable_http_ac_validation",
+			Usage:  "Whether to disable ActionResult validation for HTTP requests.  Default is false (enable validation).",
+			EnvVar: "BAZEL_REMOTE_DISABLE_HTTP_AC_VALIDATION",
+		},
 	}
 
 	app.Action = func(ctx *cli.Context) error {
@@ -166,6 +171,7 @@ func main() {
 				ctx.String("tls_key_file"),
 				ctx.Duration("idle_timeout"),
 				s3,
+				ctx.Bool("disable_http_ac_validation"),
 			)
 		}
 
@@ -207,7 +213,8 @@ func main() {
 			Addr:    c.Host + ":" + strconv.Itoa(c.Port),
 			Handler: mux,
 		}
-		h := server.NewHTTPCache(proxyCache, accessLogger, errorLogger)
+		validateAC := !c.DisableHTTPACValidation
+		h := server.NewHTTPCache(proxyCache, accessLogger, errorLogger, validateAC)
 		mux.HandleFunc("/status", h.StatusPageHandler)
 
 		cacheHandler := h.CacheHandler
diff --git a/server/grpc_ac.go b/server/grpc_ac.go
index e53cd4c..8ae1893 100644
--- a/server/grpc_ac.go
+++ b/server/grpc_ac.go
@@ -4,7 +4,6 @@ import (
 	"bytes"
 	"context"
 	"fmt"
-	"io/ioutil"
 
 	pb "github.com/bazelbuild/remote-apis/build/bazel/remote/execution/v2"
 	"github.com/golang/protobuf/proto"
@@ -25,74 +24,19 @@ func (s *grpcServer) GetActionResult(ctx context.Context,
 		return nil, err
 	}
 
-	dr, _, err := s.cache.Get(cache.AC, req.ActionDigest.Hash)
+	result, _, err := cache.GetValidatedActionResult(s.cache,
+		req.ActionDigest.Hash)
 	if err != nil {
 		s.accessLogger.Printf("%s %s %s", errorPrefix, req.ActionDigest.Hash, err)
 		return nil, status.Error(codes.Unknown, err.Error())
 	}
-	if dr == nil {
+
+	if result == nil {
 		s.accessLogger.Printf("%s %s NOT FOUND", errorPrefix, req.ActionDigest.Hash)
 		return nil, status.Error(codes.NotFound,
 			fmt.Sprintf("%s not found in AC", req.ActionDigest.Hash))
 	}
 
-	// Note: req.ActionDigest refers to the Action, not the ActionResult.
-	// There's nothing useful we can do with req.ActionDigest.SizeBytes.
-
-	data, err := ioutil.ReadAll(dr)
-	if err != nil {
-		s.accessLogger.Printf("%s %s %s", errorPrefix, req.ActionDigest.Hash, err)
-		return nil, status.Error(codes.Unknown, err.Error())
-	}
-
-	result := &pb.ActionResult{}
-	err = proto.Unmarshal(data, result)
-	if err != nil {
-		// Invalid cache item?
-		s.accessLogger.Printf("%s %s %s", errorPrefix, req.ActionDigest.Hash, err)
-		return nil, status.Error(codes.DataLoss, err.Error())
-	}
-
-	// Check that referenced blobs are available.
-
-	for _, f := range result.OutputFiles {
-		if len(f.Contents) == 0 && f.Digest.SizeBytes > 0 {
-			if !s.cache.Contains(cache.CAS, f.Digest.Hash) {
-				s.accessLogger.Printf("GRPC CAS CONTAINS %s NOT FOUND",
-					f.Digest.Hash)
-				return nil, status.Error(codes.NotFound,
-					fmt.Sprintf("%s not found in CAS", f.Digest.Hash))
-			}
-		}
-	}
-
-	for _, d := range result.OutputDirectories {
-		if !s.cache.Contains(cache.CAS, d.TreeDigest.Hash) {
-			s.accessLogger.Printf("GRPC CAS CONTAINS %s NOT FOUND",
-				d.TreeDigest.Hash)
-			return nil, status.Error(codes.NotFound,
-				fmt.Sprintf("%s not found in CAS", d.TreeDigest.Hash))
-		}
-	}
-
-	if result.StdoutDigest != nil {
-		if !s.cache.Contains(cache.CAS, result.StdoutDigest.Hash) {
-			s.accessLogger.Printf("GRPC CAS CONTAINS %s NOT FOUND",
-				result.StdoutDigest.Hash)
-			return nil, status.Error(codes.NotFound,
-				fmt.Sprintf("%s not found in CAS", result.StdoutDigest.Hash))
-		}
-	}
-
-	if result.StderrDigest != nil {
-		if !s.cache.Contains(cache.CAS, result.StderrDigest.Hash) {
-			s.accessLogger.Printf("GRPC CAS CONTAINS %s NOT FOUND",
-				result.StderrDigest.Hash)
-			return nil, status.Error(codes.NotFound,
-				fmt.Sprintf("%s not found in CAS", result.StderrDigest.Hash))
-		}
-	}
-
 	s.accessLogger.Printf("GRPC AC GET %s OK", req.ActionDigest.Hash)
 
 	return result, nil
diff --git a/server/http.go b/server/http.go
index 594e2ca..2f70d03 100644
--- a/server/http.go
+++ b/server/http.go
@@ -1,17 +1,21 @@
 package server
 
 import (
+	"bytes"
 	"encoding/json"
 	"fmt"
 	"html"
 	"io"
+	"io/ioutil"
 	"net"
 	"net/http"
 	"regexp"
 	"strconv"
 	"time"
 
+	pb "github.com/bazelbuild/remote-apis/build/bazel/remote/execution/v2"
 	"github.com/buchgr/bazel-remote/cache"
+	"github.com/golang/protobuf/proto"
 )
 
 var blobNameSHA256 = regexp.MustCompile("^/?(.*/)?(ac/|cas/)([a-f0-9]{64})$")
@@ -26,6 +30,7 @@ type httpCache struct {
 	cache        cache.Cache
 	accessLogger cache.Logger
 	errorLogger  cache.Logger
+	validateAC   bool
 }
 
 type statusPageData struct {
@@ -43,7 +48,7 @@ var GitCommit string
 // accessLogger will print one line for each HTTP request to stdout.
 // errorLogger will print unexpected server errors. Inexistent files and malformed URLs will not
 // be reported.
-func NewHTTPCache(cache cache.Cache, accessLogger cache.Logger, errorLogger cache.Logger) HTTPCache {
+func NewHTTPCache(cache cache.Cache, accessLogger cache.Logger, errorLogger cache.Logger, validateAC bool) HTTPCache {
 	if len(GitCommit) > 0 {
 		errorLogger.Printf("Server built from git commit %s.", GitCommit)
 	}
@@ -53,12 +58,13 @@ func NewHTTPCache(cache cache.Cache, accessLogger cache.Logger, errorLogger cach
 		cache:        cache,
 		accessLogger: accessLogger,
 		errorLogger:  errorLogger,
+		validateAC:   validateAC,
 	}
 	return hc
 }
 
 // Parse cache artifact information from the request URL
-func parseRequestURL(url string) (cache.EntryKind, string, error) {
+func parseRequestURL(url string, validateAC bool) (cache.EntryKind, string, error) {
 	m := blobNameSHA256.FindStringSubmatch(url)
 	if m == nil {
 		err := fmt.Errorf("resource name must be a SHA256 hash in hex. "+
@@ -78,33 +84,90 @@ func parseRequestURL(url string) (cache.EntryKind, string, error) {
 	if parts[0] == "cas/" {
 		return cache.CAS, hash, nil
 	}
-	return cache.AC, hash, nil
+
+	if validateAC {
+		return cache.AC, hash, nil
+	}
+
+	return cache.RAW, hash, nil
 }
+func (h *httpCache) handleContainsValidAC(w http.ResponseWriter, r *http.Request, hash string) {
+	_, data, err := cache.GetValidatedActionResult(h.cache, hash)
+	if err != nil {
+		http.Error(w, "Not found", http.StatusNotFound)
+		h.logResponse(http.StatusNotFound, r)
+		return
+	}
 
-func (h *httpCache) CacheHandler(w http.ResponseWriter, r *http.Request) {
-	defer r.Body.Close()
+	if data == nil {
+		http.Error(w, "Not found", http.StatusNotFound)
+		h.logResponse(http.StatusNotFound, r)
+		return
+	}
 
-	// Helper function for logging responses
-	logResponse := func(code int) {
-		// Parse the client ip:port
-		var clientAddress string
-		var err error
-		clientAddress, _, err = net.SplitHostPort(r.RemoteAddr)
-		if err != nil {
-			clientAddress = r.RemoteAddr
-		}
-		h.accessLogger.Printf("%4s %d %15s %s", r.Method, code, clientAddress, r.URL.Path)
+	w.WriteHeader(http.StatusOK)
+	h.logResponse(http.StatusOK, r)
+}
+
+func (h *httpCache) handleGetValidAC(w http.ResponseWriter, r *http.Request, hash string) {
+	_, data, err := cache.GetValidatedActionResult(h.cache, hash)
+	if err != nil {
+		http.Error(w, "Not found", http.StatusNotFound)
+		h.logResponse(http.StatusNotFound, r)
+		return
+	}
+
+	if data == nil {
+		http.Error(w, "Not found", http.StatusNotFound)
+		h.logResponse(http.StatusNotFound, r)
+		return
+	}
+
+	w.Header().Set("Content-Type", "application/octet-stream")
+	w.Header().Set("Content-Length", strconv.FormatInt(int64(len(data)), 10))
+	bytesWritten, err := w.Write(data)
+
+	if err != nil {
+		h.logResponse(http.StatusInternalServerError, r)
+		return
+	}
+
+	if bytesWritten != len(data) {
+		h.logResponse(http.StatusInternalServerError, r)
+		return
 	}
+}
 
-	kind, hash, err := parseRequestURL(r.URL.Path)
+// Helper function for logging responses
+func (h *httpCache) logResponse(code int, r *http.Request) {
+	// Parse the client ip:port
+	var clientAddress string
+	var err error
+	clientAddress, _, err = net.SplitHostPort(r.RemoteAddr)
+	if err != nil {
+		clientAddress = r.RemoteAddr
+	}
+	h.accessLogger.Printf("%4s %d %15s %s", r.Method, code, clientAddress, r.URL.Path)
+}
+
+func (h *httpCache) CacheHandler(w http.ResponseWriter, r *http.Request) {
+	defer r.Body.Close()
+
+	kind, hash, err := parseRequestURL(r.URL.Path, h.validateAC)
 	if err != nil {
 		http.Error(w, err.Error(), http.StatusBadRequest)
-		logResponse(http.StatusBadRequest)
+		h.logResponse(http.StatusBadRequest, r)
 		return
 	}
 
 	switch m := r.Method; m {
 	case http.MethodGet:
+
+		if h.validateAC && kind == cache.AC {
+			h.handleGetValidAC(w, r, hash)
+			return
+		}
+
 		rdr, sizeBytes, err := h.cache.Get(kind, hash)
 		if err != nil {
 			if e, ok := err.(*cache.Error); ok {
@@ -118,7 +181,7 @@ func (h *httpCache) CacheHandler(w http.ResponseWriter, r *http.Request) {
 
 		if rdr == nil {
 			http.Error(w, "Not found", http.StatusNotFound)
-			logResponse(http.StatusNotFound)
+			h.logResponse(http.StatusNotFound, r)
 			return
 		}
 		defer rdr.Close()
@@ -127,7 +190,7 @@ func (h *httpCache) CacheHandler(w http.ResponseWriter, r *http.Request) {
 		w.Header().Set("Content-Length", strconv.FormatInt(sizeBytes, 10))
 		io.Copy(w, rdr)
 
-		logResponse(http.StatusOK)
+		h.logResponse(http.StatusOK, r)
 	case http.MethodPut:
 		if r.ContentLength == -1 {
 			// We need the content-length header to make sure we have enough disk space.
@@ -137,7 +200,33 @@ func (h *httpCache) CacheHandler(w http.ResponseWriter, r *http.Request) {
 			return
 		}
 
-		err := h.cache.Put(kind, hash, r.ContentLength, r.Body)
+		rc := r.Body
+		if h.validateAC && kind == cache.AC {
+			// verify that this is a valid ActionResult
+
+			data, err := ioutil.ReadAll(rc)
+			if err != nil {
+				msg := "Failed to read request body"
+				http.Error(w, msg, http.StatusBadRequest) // find a better status
+				h.errorLogger.Printf("%s", msg)
+			}
+
+			ar := &pb.ActionResult{}
+			err = proto.Unmarshal(data, ar)
+			if err != nil {
+				msg := "Put received an invalid ActionResult"
+				http.Error(w, msg, http.StatusBadRequest)
+				h.errorLogger.Printf("%s", msg)
+				return
+			}
+
+			// Note: we do not currently verify that the blobs exist
+			// in the CAS.
+
+			rc = ioutil.NopCloser(bytes.NewReader(data))
+		}
+
+		err := h.cache.Put(kind, hash, r.ContentLength, rc)
 		if err != nil {
 			if cerr, ok := err.(*cache.Error); ok {
 				http.Error(w, err.Error(), cerr.Code)
@@ -146,23 +235,31 @@ func (h *httpCache) CacheHandler(w http.ResponseWriter, r *http.Request) {
 			}
 			h.errorLogger.Printf("PUT %s: %s", path(kind, hash), err)
 		} else {
-			logResponse(http.StatusOK)
+			h.logResponse(http.StatusOK, r)
 		}
 
 	case http.MethodHead:
+
+		if h.validateAC && kind == cache.AC {
+			h.handleContainsValidAC(w, r, hash)
+			return
+		}
+
+		// Unvalidated path:
+
 		ok := h.cache.Contains(kind, hash)
 		if !ok {
 			http.Error(w, "Not found", http.StatusNotFound)
-			logResponse(http.StatusNotFound)
+			h.logResponse(http.StatusNotFound, r)
 			return
 		}
 
 		w.WriteHeader(http.StatusOK)
-		logResponse(http.StatusOK)
+		h.logResponse(http.StatusOK, r)
 	default:
 		msg := fmt.Sprintf("Method '%s' not supported.", html.EscapeString(m))
 		http.Error(w, msg, http.StatusMethodNotAllowed)
-		logResponse(http.StatusMethodNotAllowed)
+		h.logResponse(http.StatusMethodNotAllowed, r)
 	}
 }
 


Test patch:
diff --git a/cache/disk/disk_test.go b/cache/disk/disk_test.go
index 0d57a04..96ac903 100644
--- a/cache/disk/disk_test.go
+++ b/cache/disk/disk_test.go
@@ -170,7 +170,14 @@ func expectContentEquals(rdr io.ReadCloser, sizeBytes int64, expectedContent []b
 }
 
 func putGetCompare(kind cache.EntryKind, hash string, content string, testCache cache.Cache) error {
-	err := testCache.Put(kind, hash, int64(len(content)), strings.NewReader(content))
+	return putGetCompareBytes(kind, hash, []byte(content), testCache)
+}
+
+func putGetCompareBytes(kind cache.EntryKind, hash string, data []byte, testCache cache.Cache) error {
+
+	r := bytes.NewReader(data)
+
+	err := testCache.Put(kind, hash, int64(len(data)), r)
 	if err != nil {
 		return err
 	}
@@ -180,7 +187,7 @@ func putGetCompare(kind cache.EntryKind, hash string, content string, testCache
 		return err
 	}
 	// Get the item back
-	return expectContentEquals(rdr, sizeBytes, []byte(content))
+	return expectContentEquals(rdr, sizeBytes, data)
 }
 
 func hashStr(content string) string {
@@ -212,6 +219,15 @@ func TestOverwrite(t *testing.T) {
 	if err != nil {
 		t.Fatal(err)
 	}
+
+	err = putGetCompare(cache.RAW, hashStr("world"), "world3", testCache)
+	if err != nil {
+		t.Fatal(err)
+	}
+	err = putGetCompare(cache.RAW, hashStr("world"), "world4", testCache)
+	if err != nil {
+		t.Fatal(err)
+	}
 }
 
 func TestCacheExistingFiles(t *testing.T) {
@@ -221,11 +237,13 @@ func TestCacheExistingFiles(t *testing.T) {
 	ensureDirExists(filepath.Join(cacheDir, "cas", "f5"))
 	ensureDirExists(filepath.Join(cacheDir, "cas", "fd"))
 	ensureDirExists(filepath.Join(cacheDir, "ac", "73"))
+	ensureDirExists(filepath.Join(cacheDir, "raw", "73"))
 
 	items := []string{
 		"cas/f5/f53b46209596d170f7659a414c9ff9f6b545cf77ffd6e1cbe9bcc57e1afacfbd",
 		"cas/fd/fdce205a735f407ae2910426611893d99ed985e3d9a341820283ea0b7d046ee3",
 		"ac/73/733e21b37cef883579a88183eed0d00cdeea0b59e1bcd77db6957f881c3a6b54",
+		"raw/73/733e21b37cef883579a88183eed0d00cdeea0b59e1bcd77db6957f881c3a6b54",
 	}
 
 	for _, it := range items {
@@ -237,10 +255,10 @@ func TestCacheExistingFiles(t *testing.T) {
 		time.Sleep(10 * time.Millisecond)
 	}
 
-	const expectedSize = 3 * int64(len(CONTENTS))
+	const expectedSize = 4 * int64(len(CONTENTS))
 	testCache := New(cacheDir, expectedSize)
 
-	err := checkItems(testCache.(*diskCache), expectedSize, 3)
+	err := checkItems(testCache.(*diskCache), expectedSize, 4)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -251,7 +269,7 @@ func TestCacheExistingFiles(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	err = checkItems(testCache.(*diskCache), expectedSize, 3)
+	err = checkItems(testCache.(*diskCache), expectedSize, 4)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -268,30 +286,41 @@ func TestCacheBlobTooLarge(t *testing.T) {
 	defer os.RemoveAll(cacheDir)
 	testCache := New(cacheDir, 100)
 
-	err := testCache.Put(cache.AC, hashStr("foo"), 10000, strings.NewReader(CONTENTS))
-	if err == nil {
-		t.Fatal("Expected an error")
-	}
+	for k := range []cache.EntryKind{cache.AC, cache.RAW} {
+		kind := cache.EntryKind(k)
+		err := testCache.Put(kind, hashStr("foo"), 10000, strings.NewReader(CONTENTS))
+		if err == nil {
+			t.Fatal("Expected an error")
+		}
 
-	if cerr, ok := err.(*cache.Error); ok {
-		if cerr.Code != http.StatusInsufficientStorage {
-			t.Fatalf("Expected error code %d but received %d", http.StatusInsufficientStorage, cerr.Code)
+		if cerr, ok := err.(*cache.Error); ok {
+			if cerr.Code != http.StatusInsufficientStorage {
+				t.Fatalf("Expected error code %d but received %d", http.StatusInsufficientStorage, cerr.Code)
+			}
+		} else {
+			t.Fatal("Expected error to be of type Error")
 		}
-	} else {
-		t.Fatal("Expected error to be of type Error")
 	}
 }
 
 // Make sure that Cache rejects an upload whose hashsum doesn't match
-func TestCacheCorruptedFile(t *testing.T) {
+func TestCacheCorruptedCASBlob(t *testing.T) {
 	cacheDir := tempDir(t)
 	defer os.RemoveAll(cacheDir)
 	testCache := New(cacheDir, 1000)
 
-	err := testCache.Put(cache.CAS, hashStr("foo"), int64(len(CONTENTS)), strings.NewReader(CONTENTS))
+	err := testCache.Put(cache.CAS, hashStr("foo"), int64(len(CONTENTS)),
+		strings.NewReader(CONTENTS))
 	if err == nil {
 		t.Fatal("expected hash mismatch error")
 	}
+
+	// We expect the upload to succeed without validation:
+	err = testCache.Put(cache.RAW, hashStr("foo"), int64(len(CONTENTS)),
+		strings.NewReader(CONTENTS))
+	if err != nil {
+		t.Fatal(err)
+	}
 }
 
 func TestMigrateFromOldDirectoryStructure(t *testing.T) {
@@ -329,18 +358,27 @@ func TestLoadExistingEntries(t *testing.T) {
 	// Test that loading existing items works
 	cacheDir := testutils.TempDir(t)
 	defer os.RemoveAll(cacheDir)
-	acHash, err := testutils.CreateCacheFile(cacheDir+"/ac/", 1024)
+
+	numBlobs := int64(3)
+	blobSize := int64(1024)
+
+	acHash, err := testutils.CreateCacheFile(cacheDir+"/ac/", blobSize)
+	if err != nil {
+		t.Fatal(err)
+	}
+	casHash, err := testutils.CreateCacheFile(cacheDir+"/cas/", blobSize)
 	if err != nil {
 		t.Fatal(err)
 	}
-	casHash, err := testutils.CreateCacheFile(cacheDir+"/cas/", 1024)
+	rawHash, err := testutils.CreateCacheFile(cacheDir+"/raw/", blobSize)
 	if err != nil {
 		t.Fatal(err)
 	}
 
-	testCache := New(cacheDir, 2048)
-	if testCache.NumItems() != 2 {
-		t.Fatalf("Expected test cache size 2 but was %d", testCache.NumItems())
+	testCache := New(cacheDir, blobSize*numBlobs)
+	if int64(testCache.NumItems()) != numBlobs {
+		t.Fatalf("Expected test cache size %d but was %d",
+			numBlobs, testCache.NumItems())
 	}
 	if !testCache.Contains(cache.AC, acHash) {
 		t.Fatalf("Expected cache to contain AC entry '%s'", acHash)
@@ -348,4 +386,45 @@ func TestLoadExistingEntries(t *testing.T) {
 	if !testCache.Contains(cache.CAS, casHash) {
 		t.Fatalf("Expected cache to contain CAS entry '%s'", casHash)
 	}
+	if !testCache.Contains(cache.RAW, rawHash) {
+		t.Fatalf("Expected cache to contain RAW entry '%s'", rawHash)
+	}
+}
+
+func TestDistinctKeyspaces(t *testing.T) {
+	cacheDir := testutils.TempDir(t)
+	defer os.RemoveAll(cacheDir)
+
+	blobSize := 1024
+	cacheSize := int64(blobSize * 3)
+
+	testCache := New(cacheDir, cacheSize)
+
+	blob, casHash := testutils.RandomDataAndHash(1024)
+
+	// Add the same blob with the same key, to each of the three
+	// keyspaces, and verify that we have exactly three items in
+	// the cache.
+
+	var err error
+
+	err = putGetCompareBytes(cache.CAS, casHash, blob, testCache)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	err = putGetCompareBytes(cache.AC, casHash, blob, testCache)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	err = putGetCompareBytes(cache.RAW, casHash, blob, testCache)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if testCache.NumItems() != 3 {
+		t.Fatalf("Expected test cache size 3 but was %d",
+			testCache.NumItems())
+	}
 }
diff --git a/config/config_test.go b/config/config_test.go
index 6eb5cd9..8767704 100644
--- a/config/config_test.go
+++ b/config/config_test.go
@@ -17,6 +17,7 @@ max_size: 100
 htpasswd_file: /opt/.htpasswd
 tls_cert_file: /opt/tls.cert
 tls_key_file:  /opt/tls.key
+disable_http_ac_validation: true
 `
 
 	config, err := newFromYaml([]byte(yaml))
@@ -25,14 +26,15 @@ tls_key_file:  /opt/tls.key
 	}
 
 	expectedConfig := &Config{
-		Host:         "localhost",
-		Port:         8080,
-		GRPCPort:     9092,
-		Dir:          "/opt/cache-dir",
-		MaxSize:      100,
-		HtpasswdFile: "/opt/.htpasswd",
-		TLSCertFile:  "/opt/tls.cert",
-		TLSKeyFile:   "/opt/tls.key",
+		Host:                    "localhost",
+		Port:                    8080,
+		GRPCPort:                9092,
+		Dir:                     "/opt/cache-dir",
+		MaxSize:                 100,
+		HtpasswdFile:            "/opt/.htpasswd",
+		TLSCertFile:             "/opt/tls.cert",
+		TLSKeyFile:              "/opt/tls.key",
+		DisableHTTPACValidation: true,
 	}
 
 	if !reflect.DeepEqual(config, expectedConfig) {
diff --git a/server/http_test.go b/server/http_test.go
index af694f8..ec99258 100644
--- a/server/http_test.go
+++ b/server/http_test.go
@@ -32,7 +32,7 @@ func TestDownloadFile(t *testing.T) {
 	}
 
 	c := disk.New(cacheDir, 1024)
-	h := NewHTTPCache(c, testutils.NewSilentLogger(), testutils.NewSilentLogger())
+	h := NewHTTPCache(c, testutils.NewSilentLogger(), testutils.NewSilentLogger(), true)
 
 	req, err := http.NewRequest("GET", "/cas/"+hash, bytes.NewReader([]byte{}))
 	if err != nil {
@@ -83,7 +83,7 @@ func TestUploadFilesConcurrently(t *testing.T) {
 	}
 
 	c := disk.New(cacheDir, 1000*1024)
-	h := NewHTTPCache(c, testutils.NewSilentLogger(), testutils.NewSilentLogger())
+	h := NewHTTPCache(c, testutils.NewSilentLogger(), testutils.NewSilentLogger(), true)
 	handler := http.HandlerFunc(h.CacheHandler)
 
 	var wg sync.WaitGroup
@@ -139,7 +139,7 @@ func TestUploadSameFileConcurrently(t *testing.T) {
 	data, hash := testutils.RandomDataAndHash(1024)
 
 	c := disk.New(cacheDir, 1024)
-	h := NewHTTPCache(c, testutils.NewSilentLogger(), testutils.NewSilentLogger())
+	h := NewHTTPCache(c, testutils.NewSilentLogger(), testutils.NewSilentLogger(), true)
 	handler := http.HandlerFunc(h.CacheHandler)
 
 	var wg sync.WaitGroup
@@ -184,7 +184,7 @@ func TestUploadCorruptedFile(t *testing.T) {
 	}
 
 	c := disk.New(cacheDir, 2048)
-	h := NewHTTPCache(c, testutils.NewSilentLogger(), testutils.NewSilentLogger())
+	h := NewHTTPCache(c, testutils.NewSilentLogger(), testutils.NewSilentLogger(), true)
 	rr := httptest.NewRecorder()
 	handler := http.HandlerFunc(h.CacheHandler)
 	handler.ServeHTTP(rr, r)
@@ -222,7 +222,7 @@ func TestStatusPage(t *testing.T) {
 	}
 
 	c := disk.New(cacheDir, 2048)
-	h := NewHTTPCache(c, testutils.NewSilentLogger(), testutils.NewSilentLogger())
+	h := NewHTTPCache(c, testutils.NewSilentLogger(), testutils.NewSilentLogger(), true)
 	rr := httptest.NewRecorder()
 	handler := http.HandlerFunc(h.StatusPageHandler)
 	handler.ServeHTTP(rr, r)
@@ -248,7 +248,7 @@ func TestStatusPage(t *testing.T) {
 
 func TestParseRequestURL(t *testing.T) {
 	{
-		_, _, err := parseRequestURL("invalid/url")
+		_, _, err := parseRequestURL("invalid/url", true)
 		if err == nil {
 			t.Error("Failed to reject an invalid URL")
 		}
@@ -257,7 +257,7 @@ func TestParseRequestURL(t *testing.T) {
 	const aSha256sum = "fec3be77b8aa0d307ed840581ded3d114c86f36d4914c81e33a72877020c0603"
 
 	{
-		kind, hash, err := parseRequestURL("cas/" + aSha256sum)
+		kind, hash, err := parseRequestURL("cas/"+aSha256sum, true)
 		if err != nil {
 			t.Error("Failed to parse a valid CAS URL")
 		}
@@ -265,12 +265,12 @@ func TestParseRequestURL(t *testing.T) {
 			t.Error("Cache key parsed incorrectly")
 		}
 		if kind != cache.CAS {
-			t.Errorf("Expected kind CAS but got AC")
+			t.Errorf("Expected kind CAS but got %s", kind)
 		}
 	}
 
 	{
-		kind, hash, err := parseRequestURL("ac/" + aSha256sum)
+		kind, hash, err := parseRequestURL("ac/"+aSha256sum, true)
 		if err != nil {
 			t.Error("Failed to parse a valid AC URL")
 		}
@@ -278,12 +278,12 @@ func TestParseRequestURL(t *testing.T) {
 			t.Error("Cache key parsed incorrectly")
 		}
 		if kind != cache.AC {
-			t.Error("Expected kind AC but got CAS")
+			t.Errorf("Expected kind AC but got %s", kind)
 		}
 	}
 
 	{
-		kind, hash, err := parseRequestURL("prefix/ac/" + aSha256sum)
+		kind, hash, err := parseRequestURL("prefix/ac/"+aSha256sum, true)
 		if err != nil {
 			t.Error("Failed to parse a valid AC URL with prefix")
 		}
@@ -291,7 +291,20 @@ func TestParseRequestURL(t *testing.T) {
 			t.Error("Cache key parsed incorrectly")
 		}
 		if kind != cache.AC {
-			t.Error("Expected kind AC but got CAS")
+			t.Errorf("Expected kind AC but got %s", kind)
+		}
+	}
+
+	{
+		kind, hash, err := parseRequestURL("prefix/ac/"+aSha256sum, false)
+		if err != nil {
+			t.Error("Failed to parse a valid AC URL with prefix")
+		}
+		if hash != aSha256sum {
+			t.Error("Cache key parsed incorrectly")
+		}
+		if kind != cache.RAW {
+			t.Errorf("Expected kind RAW but got %s", kind)
 		}
 	}
 }
@@ -345,7 +358,7 @@ func (r fakeResponseWriter) WriteHeader(statusCode int) {
 
 func TestRemoteReturnsNotFound(t *testing.T) {
 	fake := &fakeCache{}
-	h := NewHTTPCache(fake, testutils.NewSilentLogger(), testutils.NewSilentLogger())
+	h := NewHTTPCache(fake, testutils.NewSilentLogger(), testutils.NewSilentLogger(), true)
 	// create a fake http.Request
 	_, hash := testutils.RandomDataAndHash(1024)
 	url, _ := url.Parse(fmt.Sprintf("http://localhost:8080/ac/%s", hash))