Repository: decred/dcrd
Pull request #3145 (2023-06-14 18:00:55)

build: Add CI support for test and module cache.
This makes use of the github actions cache to save and restore the Go test and module caches.  This should result in faster CI runs since most changes only affect a small number of tests.

It also updates the RPC integration tests to ensure the logs directory is not in the source code dir so the go testing cache can be used.

Next, it renames the `txscript` module's `data` dir to `testdata` so `go` does not treat it like a package.

Finally, it adds a script that stabilizes the timestamps on all files in `testdata` directories when running the github action.  This is necessary because the go testing cache logic uses the timestamps of all input files when determining whether or not a test needs to be rerun and github clones a fresh repo on every run which causes the timestamps of the test data to change.

davecgh: CI is now completing in about a minute with these changes.

Patch:
diff --git a/.github/workflows/go.yml b/.github/workflows/go.yml
index d92989ba..94cba0e0 100644
--- a/.github/workflows/go.yml
+++ b/.github/workflows/go.yml
@@ -19,6 +19,17 @@ jobs:
         uses: actions/checkout@8e5e7e5ab8b370d6c329ec480221332ada57f0ab #v3.5.2
       - name: Install Linters
         run: "curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(go env GOPATH)/bin v1.53.1"
+      - name: Use test and module cache
+        uses: actions/cache@88522ab9f39a2ea568f7027eddc7d8d8bc9d59c8 #v3.3.1
+        with:
+          path: |
+            ~/.cache/go-build
+            ~/go/pkg/mod
+          key: go-test-${{ matrix.go }}-${{ github.sha }}
+          restore-keys: go-test-${{ matrix.go }}
+      - name: Stablilize testdata timestamps
+        run: |
+          bash ./.github/stablilize_testdata_timestamps.sh "${{ github.workspace }}"
       - name: Build
         run: go build ./...
       - name: Test


Test patch:
diff --git a/.github/stablilize_testdata_timestamps.sh b/.github/stablilize_testdata_timestamps.sh
new file mode 100644
index 00000000..4b0379fc
--- /dev/null
+++ b/.github/stablilize_testdata_timestamps.sh
@@ -0,0 +1,24 @@
+#!/bin/env bash
+
+display_usage() {
+	echo -e "Usage: $0 PATH_TO_REPOSITORY_ROOT\n"
+}
+
+# Check arguments and set vars with them.
+if [ $# -ne 1 ]; then
+        display_usage
+        exit 1
+fi
+REPO_ROOT="${1%%/}"
+
+basetime=$(date -d "2023-06-14" +%s)
+for f in $(find "${REPO_ROOT}" -type f -path "*/testdata/*"); do
+  # Update the file timestamp based on the hash of the contents of the file.
+  # This ensures the cache is busted when the contents change while still
+  # providing a stable timestamp.
+  filehash=$(sha256sum "$f" | tr '[:lower:]' '[:upper:]')
+  hashmod10k=$(echo "ibase=16; $(echo $filehash | cut -c1-10) % 2710" | bc)
+  modtime=$(( $basetime - $hashmod10k ))
+  touch -d "@$modtime" "$f"
+  echo "$f timestamp updated to $modtime"
+done
diff --git a/internal/integration/rpctests/treasury_test.go b/internal/integration/rpctests/treasury_test.go
index f3e59c72..3f178dd6 100644
--- a/internal/integration/rpctests/treasury_test.go
+++ b/internal/integration/rpctests/treasury_test.go
@@ -14,6 +14,7 @@ import (
 	"encoding/binary"
 	"encoding/hex"
 	"os"
+	"path/filepath"
 	"testing"
 	"time"
 
@@ -211,7 +212,8 @@ func TestTreasury(t *testing.T) {
 	defaultFeeRate := dcrutil.Amount(1e4)
 
 	// Setup the log dir for tests to ease debugging after failures.
-	logDir := ".dcrdlogs"
+	logDir := filepath.Join(os.TempDir(), t.Name(), ".dcrdlogs")
+	t.Logf("Logdir: %s", logDir)
 	extraArgs := []string{
 		"--rejectnonstd",
 		"--debuglevel=MINR=trace,TRSY=trace",
diff --git a/txscript/bench_test.go b/txscript/bench_test.go
index 41cb9481..e7676726 100644
--- a/txscript/bench_test.go
+++ b/txscript/bench_test.go
@@ -8,6 +8,7 @@ import (
 	"bytes"
 	"fmt"
 	"os"
+	"path/filepath"
 	"testing"
 
 	"github.com/decred/dcrd/wire"
@@ -36,7 +37,8 @@ var (
 
 func init() {
 	// tx 620f57c92cf05a7f7e7f7d28255d5f7089437bc48e34dcfebf7751d08b7fb8f5
-	txHex, err := os.ReadFile("data/many_inputs_tx.hex")
+	inputFilePath := filepath.Join(testDataPath, "many_inputs_tx.hex")
+	txHex, err := os.ReadFile(inputFilePath)
 	if err != nil {
 		panic(fmt.Sprintf("unable to read benchmark tx file: %v", err))
 	}
diff --git a/txscript/reference_test.go b/txscript/reference_test.go
index 8571acbc..59feea94 100644
--- a/txscript/reference_test.go
+++ b/txscript/reference_test.go
@@ -12,6 +12,7 @@ import (
 	"errors"
 	"fmt"
 	"os"
+	"path/filepath"
 	"strings"
 	"testing"
 
@@ -316,7 +317,7 @@ func testScripts(t *testing.T, tests [][]string, useSigCache bool) {
 // TestScripts ensures all of the tests in script_tests.json execute with the
 // expected results as defined in the test data.
 func TestScripts(t *testing.T) {
-	file, err := os.ReadFile("data/script_tests.json")
+	file, err := os.ReadFile(filepath.Join(testDataPath, "script_tests.json"))
 	if err != nil {
 		t.Fatalf("TestScripts: %v\n", err)
 	}
@@ -347,7 +348,7 @@ func testVecF64ToUint32(f float64) uint32 {
 // TestTxInvalidTests ensures all of the tests in tx_invalid.json fail as
 // expected.
 func TestTxInvalidTests(t *testing.T) {
-	file, err := os.ReadFile("data/tx_invalid.json")
+	file, err := os.ReadFile(filepath.Join(testDataPath, "tx_invalid.json"))
 	if err != nil {
 		t.Errorf("TestTxInvalidTests: %v\n", err)
 		return
@@ -487,7 +488,7 @@ testloop:
 
 // TestTxValidTests ensures all of the tests in tx_valid.json pass as expected.
 func TestTxValidTests(t *testing.T) {
-	file, err := os.ReadFile("data/tx_valid.json")
+	file, err := os.ReadFile(filepath.Join(testDataPath, "tx_valid.json"))
 	if err != nil {
 		t.Errorf("TestTxValidTests: %v\n", err)
 		return
@@ -642,7 +643,7 @@ func parseSigHashExpectedResult(expected string) (error, error) {
 // TestCalcSignatureHashReference runs the reference signature hash calculation
 // tests in sighash.json.
 func TestCalcSignatureHashReference(t *testing.T) {
-	file, err := os.ReadFile("data/sighash.json")
+	file, err := os.ReadFile(filepath.Join(testDataPath, "sighash.json"))
 	if err != nil {
 		t.Fatalf("TestCalcSignatureHash: %v\n", err)
 	}
diff --git a/txscript/sigcache_test.go b/txscript/sigcache_test.go
index 96e8f445..d4bb1593 100644
--- a/txscript/sigcache_test.go
+++ b/txscript/sigcache_test.go
@@ -19,7 +19,7 @@ import (
 )
 
 // testDataPath is the path where txscript test fixtures reside.
-const testDataPath = "data"
+const testDataPath = "testdata"
 
 // block432100 mocks block 432,100 of the block chain.  It is loaded and
 // deserialized immediately here and then can be used throughout the tests.