Repository: buchgr/bazel-remote
Pull request #411 (2021-03-22 06:50:56)

more v1->v2 migration fixes


Patch:
diff --git a/.bazelci/presubmit.yml b/.bazelci/presubmit.yml
index 56c3c37..ef81625 100644
--- a/.bazelci/presubmit.yml
+++ b/.bazelci/presubmit.yml
@@ -61,3 +61,9 @@ tasks:
     shell_commands:
       - "echo +++ Run TLS tests"
       - ".bazelci/tls-tests.sh"
+  migration_tests:
+    platform: ubuntu2004
+    name: "migration tests"
+    shell_commands:
+      - "echo +++ Run migration tests"
+      - ".bazelci/migration-test.sh"
diff --git a/cache/disk/disk.go b/cache/disk/disk.go
index 36959b5..b94e892 100644
--- a/cache/disk/disk.go
+++ b/cache/disk/disk.go
@@ -341,16 +341,14 @@ func migrateV1Subdir(oldDir string, destDir string, kind cache.EntryKind) error
 		return os.Remove(oldDir)
 	}
 
-	var hashKeyRegexWithRandom = regexp.MustCompile("^([a-f0-9]{64})-([0-9]+)$")
-
 	for _, item := range listing {
 		oldPath := path.Join(oldDir, item.Name())
 
-		if !hashKeyRegexWithRandom.MatchString(item.Name()) {
-			return fmt.Errorf("Unexpected file: %s", oldPath)
+		if !hashKeyRegex.MatchString(item.Name()) {
+			return fmt.Errorf("Unexpected file: %s %s", oldPath, item.Name())
 		}
 
-		destPath := path.Join(destDir, item.Name())
+		destPath := path.Join(destDir, item.Name()) + "-112233"
 
 		// TODO: support cross-filesystem migration.
 		err = os.Rename(oldPath, destPath)


Test patch:
diff --git a/.bazelci/migration-test.sh b/.bazelci/migration-test.sh
new file mode 100755
index 0000000..392be36
--- /dev/null
+++ b/.bazelci/migration-test.sh
@@ -0,0 +1,80 @@
+#!/usr/bin/env bash
+
+set -v
+set -x
+set -e
+set -u
+set -o pipefail
+
+SRC_ROOT=$(dirname "$0")/..
+SRC_ROOT=$(realpath "$SRC_ROOT")
+cd "$SRC_ROOT"
+
+tmpdir=$(mktemp -d bazel-remote-migration-tests.XXXXXXX --tmpdir=${TMPDIR:-/tmp})
+
+if [ ! -e bazel-remote-1.3.1-linux-x86_64 ]
+then
+	# Download a v1 bazel-remote build.
+	wget https://github.com/buchgr/bazel-remote/releases/download/v1.3.1/bazel-remote-1.3.1-linux-x86_64
+	chmod +x bazel-remote-1.3.1-linux-x86_64
+fi
+
+# Build a v2 bazel-remote binary
+[ -e bazel-remote ] || ./linux-build.sh
+
+# Run the v1 bazel-remote.
+./bazel-remote-1.3.1-linux-x86_64 --dir "$tmpdir/cache" --max_size 1 \
+	--port 8089 > "$tmpdir/v1.log" 2>&1 &
+server_pid=$!
+
+# Let the server start up...
+sleep 2
+
+# Populate the cache.
+bazel clean
+bazel build //:bazel-remote --remote_cache=grpc://localhost:9092
+
+if ! kill -0 "$server_pid"
+then
+	echo "Error: bazel-remote no longer running. Check $tmpdir/v1.log"
+	exit 1
+fi
+
+# Stop the v1 bazel-remote.
+kill -9 $server_pid
+sleep 1
+
+# Count the number of files in the cache.
+find "$tmpdir/cache" -type f | sort > "$tmpdir/v1.files"
+count_v1=$(wc -l < "$tmpdir/v1.files")
+
+# Start the v2 bazel-remote.
+./bazel-remote --dir "$tmpdir/cache" --max_size 1 \
+	--port 8089 > "$tmpdir/v2.log" 2>&1 &
+server_pid=$!
+
+# Let the server start up...
+sleep 5
+
+if ! kill -0 "$server_pid"
+then
+	echo "Error: bazel-remote no longer running. Check $tmpdir/v2.log"
+	exit 1
+fi
+
+kill -9 $server_pid
+
+# Count the number of files in the cache.
+find "$tmpdir/cache" -type f | sort > "$tmpdir/v2.files"
+count_v2=$(wc -l < "$tmpdir/v2.files")
+
+if [ "$count_v1" != "$count_v2" ]
+then
+	echo "Error: v1 directory contained \"$count_v1\" files, but migrated v2 directory contains \"$count_v2\""
+
+	rm -rf "$tmpdir"
+	exit 1
+fi
+
+# Clean up...
+rm -rf "$tmpdir"
diff --git a/cache/disk/disk_test.go b/cache/disk/disk_test.go
index d216391..65115cf 100644
--- a/cache/disk/disk_test.go
+++ b/cache/disk/disk_test.go
@@ -362,13 +362,13 @@ func TestCacheExistingFiles(t *testing.T) {
 			"foo",
 			"733e21b37cef883579a88183eed0d00cdeea0b59e1bcd77db6957f881c3a6b54",
 			"ac/733e21b37cef883579a88183eed0d00cdeea0b59e1bcd77db6957f881c3a6b54",
-			"ac/73/733e21b37cef883579a88183eed0d00cdeea0b59e1bcd77db6957f881c3a6b54-123456789",
+			"ac.v2/73/733e21b37cef883579a88183eed0d00cdeea0b59e1bcd77db6957f881c3a6b54-123456789",
 		},
 		{
 			"bar",
 			"733e21b37cef883579a88183eed0d00cdeea0b59e1bcd77db6957f881c3a6b54",
 			"raw/733e21b37cef883579a88183eed0d00cdeea0b59e1bcd77db6957f881c3a6b54",
-			"raw/73/733e21b37cef883579a88183eed0d00cdeea0b59e1bcd77db6957f881c3a6b54-123456789",
+			"raw.v2/73/733e21b37cef883579a88183eed0d00cdeea0b59e1bcd77db6957f881c3a6b54-123456789",
 		},
 	}
 
@@ -597,11 +597,12 @@ func TestLoadExistingEntries(t *testing.T) {
 	cacheDir := testutils.TempDir(t)
 	defer os.RemoveAll(cacheDir)
 
-	numBlobs := int64(4)
+	numBlobs := int64(5)
 	blobSize := int64(1024)
 
 	var err error
 
+	// V0 AC entry.
 	acData, acHash := testutils.RandomDataAndHash(blobSize)
 	err = os.MkdirAll(path.Join(cacheDir, "ac"), 0755)
 	if err != nil {
@@ -634,12 +635,24 @@ func TestLoadExistingEntries(t *testing.T) {
 		t.Fatal(err)
 	}
 
+	// V1 AC entry.
+	acV1Data, acV1Hash := testutils.RandomDataAndHash(blobSize)
+	err = os.MkdirAll(path.Join(cacheDir, "ac", acV1Hash[:2]), 0755)
+	if err != nil {
+		t.Fatal(err)
+	}
+	err = ioutil.WriteFile(path.Join(cacheDir, "ac", acV1Hash[:2], acV1Hash), acV1Data, 0644)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// V1 RAW entry.
 	rawData, rawHash := testutils.RandomDataAndHash(blobSize)
-	err = os.MkdirAll(path.Join(cacheDir, "raw"), 0755)
+	err = os.MkdirAll(path.Join(cacheDir, "raw", rawHash[:2]), 0755)
 	if err != nil {
 		t.Fatal(err)
 	}
-	err = ioutil.WriteFile(path.Join(cacheDir, "raw", rawHash), rawData, 0644)
+	err = ioutil.WriteFile(path.Join(cacheDir, "raw", rawHash[:2], rawHash), rawData, 0644)
 	if err != nil {
 		t.Fatal(err)
 	}