Repository: cockroachdb/pebble (go)

Issue:
reorganize CI and nightly builds
Currently, we run a 22 test builds on each PR:
```
Test / go-build-other (1.22)
Test / go-build-other (1.23)
Test / go-lint-checks (1.22)
Test / go-lint-checks (1.23)
Test / go-lint-checks
Test / go-linux (1.22)
Test / go-linux (1.23)
Test / go-linux
Test / go-linux-32bit (1.22)
Test / go-linux-32bit (1.23)
Test / go-linux-crossversion (1.22)
Test / go-linux-crossversion (1.23)
Test / go-linux-no-cgo (1.22)
Test / go-linux-no-cgo (1.23)
Test / go-linux-no-invariants (1.22)
Test / go-linux-no-invariants (1.23)
Test / go-linux-race (1.22)
Test / go-linux-race (1.23)
Test / go-macos (1.22)
Test / go-macos (1.23)
Test / go-windows (1.22)
Test / go-windows (1.23)
```

In addition, we run a nightly asan/msan build.

As we add more targets - in particular s390x which emulated and is very slow, we need to move most of these targets to running only nightly.

I propose that on each PR we run:
 - go-linux (with the go.mod Go version)
 - go-linux-crossversion (with the go.mod Go version)
 - go-lint-checks

We move all the rest to a nightly workflow, which uses a matrix of Go versions. We also add logic for auto-filing of issues on nightly failures.


Jira issue: PEBBLE-348

Pull request:
github: revamp nightlies and CI
This set of changes adds many nightly tests and reduces the tests run on each PR.

Fixes #4372

#### github: add issue posting to stress nightly


#### github: consolidate instrumented nightly runs

This change adds testrace to the asan/msan runs and adds automatic
issue posting.

#### github: add nightly test matrix

Copy the current CI tests to a nightly workflow, and add a `linux-arm`
job.

#### github: pare down the tests running on each PR

We now run the two required checks plus the macos and no invariant
runs. We only run them on the Go version CRDB currently uses.

Patch:
diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml
index 74bfc8a7..17dfb7ef 100644
--- a/.github/workflows/ci.yaml
+++ b/.github/workflows/ci.yaml
@@ -3,202 +3,73 @@ name: Test
 on:
   push:
     branches:
-    - master
-    - crl-release-*
+      - master
+      - crl-release-*
   pull_request:
     branches:
-    - master
-    - crl-release-*
+      - master
+      - crl-release-*
 
 jobs:
 
-  linux:
+  # This check is required to merge PRs.
+  go-linux:
     name: go-linux
-    strategy:
-      matrix:
-        go: ['1.22', '1.23']
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
 
       - name: Set up Go
         uses: actions/setup-go@v4
         with:
-          go-version: ${{ matrix.go }}
+          go-version: '1.23'
 
       - run: GOTRACEBACK=all make test testobjiotracing generate
 
       - name: Assert workspace clean
         run: scripts/check-workspace-clean.sh
 
-  # Summary job that is required to merge PRs.
-  go-linux:
-    needs: [linux]  # Waits for all matrix jobs to complete
-    runs-on: ubuntu-latest
-    if: ${{ success() }}  # Only runs if all dependent jobs succeed
-    steps:
-      - name: Summary
-        run: echo "All go-linux matrix jobs passed successfully."
-
-  linux-32bit:
-    name: go-linux-32bit
-    strategy:
-      matrix:
-        go: ['1.22', '1.23']
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v2
-
-      - name: Set up Go
-        uses: actions/setup-go@v4
-        with:
-          go-version: ${{ matrix.go }}
-
-      - run: GOTRACEBACK=all GOARCH=386 make test
-
-  linux-crossversion:
-    name: go-linux-crossversion
-    strategy:
-      matrix:
-        go: ['1.22', '1.23']
+  # This check is required to merge PRs.
+  go-lint-checks:
+    name: go-lint-checks
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
 
       - name: Set up Go
         uses: actions/setup-go@v4
         with:
-          go-version: ${{ matrix.go }}
-
-      - run: make crossversion-meta
-
-  linux-race:
-    name: go-linux-race
-    strategy:
-      matrix:
-        go: ['1.22', '1.23']
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v2
+          go-version: '1.23'
 
-      - name: Set up Go
-        uses: actions/setup-go@v4
-        with:
-          go-version: ${{ matrix.go }}
+      - name: mod-tidy-check
+        run: make mod-tidy-check
 
-      - run: GOTRACEBACK=all make testrace TAGS=
+      - name: format-check
+        run: make format-check
 
   linux-no-invariants:
     name: go-linux-no-invariants
-    strategy:
-      matrix:
-        go: ['1.22', '1.23']
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
 
       - name: Set up Go
         uses: actions/setup-go@v4
         with:
-          go-version: ${{ matrix.go }}
+          go-version: '1.23'
 
       - run: GOTRACEBACK=all make test TAGS=
 
-  linux-no-cgo:
-    name: go-linux-no-cgo
-    strategy:
-      matrix:
-        go: ['1.22', '1.23']
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v2
-
-      - name: Set up Go
-        uses: actions/setup-go@v4
-        with:
-          go-version: ${{ matrix.go }}
-
-      - run: GOTRACEBACK=all CGO_ENABLED=0 make test TAGS=
-
   darwin:
     name: go-macos
-    strategy:
-      matrix:
-        go: ['1.22', '1.23']
     runs-on: macos-14
     steps:
-      - uses: actions/checkout@v2
-  
-      - name: Set up Go
-        uses: actions/setup-go@v4
-        with:
-          go-version: ${{ matrix.go }}
-  
-      - run: GOTRACEBACK=all make test
-
-  windows:
-    name: go-windows
-    strategy:
-      matrix:
-        go: ['1.22', '1.23']
-    runs-on: windows-latest
-    steps:
-      - uses: actions/checkout@v2
-
-      - name: Set up Go
-        uses: actions/setup-go@v4
-        with:
-          go-version: ${{ matrix.go }}
-
-      - run: go test -tags invariants ./...
-
-  lint-checks:
-    name: go-lint-checks
-    strategy:
-      matrix:
-        go: ['1.22', '1.23']
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
 
       - name: Set up Go
         uses: actions/setup-go@v4
         with:
-          go-version: ${{ matrix.go }}
+          go-version: '1.23'
 
-      - name: mod-tidy-check
-        run: make mod-tidy-check
-
-      - name: format-check
-        run: make format-check
-
-  # Summary job that is required to merge PRs.
-  go-lint-checks:
-    needs: [lint-checks]  # Waits for all matrix jobs to complete
-    runs-on: ubuntu-latest
-    if: ${{ success() }}  # Only runs if all dependent jobs succeed
-    steps:
-      - name: Summary
-        run: echo "All go-lint-checks matrix jobs passed successfully."
-
-  # Builds for other OSes and architectures.
-  go-build-other:
-    name: go-build-other
-    strategy:
-      matrix:
-        go: ['1.22', '1.23']
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v2
+      - run: GOTRACEBACK=all make test
 
-      - name: Set up Go
-        uses: actions/setup-go@v4
-        with:
-          go-version: ${{ matrix.go }}
-
-      - run: GOARCH=mips go build ./...
-      - run: GOARCH=mipsle go build ./...
-      - run: GOARCH=mips64le go build ./...
-      - run: GOOS=freebsd go build -v ./...
-      - run: GOOS=netbsd go build -v ./...
-      - run: GOOS=openbsd go build -v ./...
diff --git a/.github/workflows/nightly-instrumented.yaml b/.github/workflows/nightly-instrumented.yaml
new file mode 100644
index 00000000..f93a734f
--- /dev/null
+++ b/.github/workflows/nightly-instrumented.yaml
@@ -0,0 +1,77 @@
+name: Nightly instrumented tests
+
+on:
+  schedule:
+    - cron: "0 7 * * *" # 7am UTC, daily.
+  workflow_dispatch:
+
+jobs:
+  linux-race:
+    name: linux-race
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Set up Go
+        uses: actions/setup-go@v4
+        with:
+          go-version: "1.23"
+
+      - run: GOTRACEBACK=all make testrace TAGS=
+
+      - name: Post issue on failure
+        if: failure()
+        id: create-or-update-unique-issue
+        uses: ./.github/actions/post-issue
+        with:
+          title: "nightly testrace failed"
+          body: "The nightly testrace run failed on ${{ github.sha }}. Please review the run [here](${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }})."
+          unique-title-includes: "nightly testrace failed"
+          labels: "C-test-failure"
+
+  linux-asan:
+    name: linux-asan
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Set up Go
+        uses: actions/setup-go@v4
+        with:
+          go-version: "1.23"
+
+      - run: make testasan
+
+      - name: Post issue on failure
+        if: failure()
+        id: create-or-update-unique-issue
+        uses: ./.github/actions/post-issue
+        with:
+          title: "nightly testasan failed"
+          body: "The nightly asan test run failed on ${{ github.sha }}. Please review the run [here](${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }})."
+          unique-title-includes: "nightly testasan failed"
+          labels: "C-test-failure"
+
+  linux-msan:
+    name: linux-msan
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Set up Go
+        uses: actions/setup-go@v4
+        with:
+          go-version: "1.23"
+
+      - run: make testmsan
+
+      - name: Post issue on failure
+        if: failure()
+        id: create-or-update-unique-issue
+        uses: ./.github/actions/post-issue
+        with:
+          title: "nightly testmsan failed"
+          body: "The nightly msan test run failed on ${{ github.sha }}. Please review the run [here](${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }})."
+          unique-title-includes: "nightly testmsan failed"
+          labels: "C-test-failure"
+
diff --git a/.github/workflows/nightly-stress.yaml b/.github/workflows/nightly-stress.yaml
new file mode 100644
index 00000000..e6d6a75a
--- /dev/null
+++ b/.github/workflows/nightly-stress.yaml
@@ -0,0 +1,34 @@
+name: Nightly stress test
+
+on:
+  schedule:
+    - cron: '00 10 * * * ' # 10am UTC daily
+  workflow_dispatch:
+
+jobs:
+
+  linux-stress:
+    name: linux-stress
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Set up Go
+        uses: actions/setup-go@v4
+        with:
+          go-version: "1.23"
+
+      - name: Run unit tests under stress
+        run: |
+          go install github.com/cockroachdb/stress@latest
+          scripts/stress.sh
+
+      - name: Post issue on failure
+        if: failure()
+        id: create-or-update-unique-issue
+        uses: ./.github/actions/post-issue
+        with:
+          title: "nightly stress tests failed"
+          body: "The nightly tress test run failed on ${{ github.sha }}. Please review the run [here](${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }})."
+          unique-title-includes: "nightly stress tests failed"
+          labels: "C-test-failure"
diff --git a/.github/workflows/sanitizers.yaml b/.github/workflows/sanitizers.yaml
deleted file mode 100644
index 0cade355..00000000
--- a/.github/workflows/sanitizers.yaml
+++ /dev/null
@@ -1,32 +0,0 @@
-name: Sanitizers
-
-on:
-  schedule:
-  - cron: "0 0 * * *" # Midnight UTC, daily.
-
-jobs:
-  linux-asan:
-    name: go-linux-asan
-    runs-on: ubuntu-latest
-    steps:
-    - uses: actions/checkout@v2
-
-    - name: Set up Go
-      uses: actions/setup-go@v2
-      with:
-        go-version: "1.23"
-
-    - run: make testasan
-
-  linux-msan:
-    name: go-linux-msan
-    runs-on: ubuntu-latest
-    steps:
-    - uses: actions/checkout@v2
-
-    - name: Set up Go
-      uses: actions/setup-go@v2
-      with:
-        go-version: "1.23"
-
-    - run: make testmsan
diff --git a/.github/workflows/stress.yaml b/.github/workflows/stress.yaml
deleted file mode 100644
index f41c2a64..00000000
--- a/.github/workflows/stress.yaml
+++ /dev/null
@@ -1,24 +0,0 @@
-name: Nightly stress test
-
-on:
-  schedule:
-    - cron: '00 10 * * * '
-  workflow_dispatch:
-
-jobs:
-
-  linux-stress:
-    name: linux-stress
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v2
-
-      - name: Set up Go
-        uses: actions/setup-go@v4
-        with:
-          go-version: "1.23"
-
-      - name: Run unit tests under stress
-        run: |
-          go install github.com/cockroachdb/stress@latest
-          scripts/stress.sh
diff --git a/Makefile b/Makefile
index 0cd65e3a..51155a22 100644
--- a/Makefile
+++ b/Makefile
@@ -70,7 +70,7 @@ crossversion-meta:
 		${GO} test -c ./internal/metamorphic -o './internal/metamorphic/crossversion/${LATEST_RELEASE}.test'; \
 		git checkout -; \
 		${GO} test -c ./internal/metamorphic -o './internal/metamorphic/crossversion/head.test'; \
-		${GO} test -tags '$(TAGS)' ${testflags} -v -run 'TestMetaCrossVersion' ./internal/metamorphic/crossversion --version '${LATEST_RELEASE},${LATEST_RELEASE},${LATEST_RELEASE}.test' --version 'HEAD,HEAD,./head.test'
+		${GO} test -tags '$(TAGS)' ${testflags} -v -timeout 20m -run 'TestMetaCrossVersion' ./internal/metamorphic/crossversion --version '${LATEST_RELEASE},${LATEST_RELEASE},${LATEST_RELEASE}.test' --version 'HEAD,HEAD,./head.test'
 
 .PHONY: stress-crossversion
 stress-crossversion:


Test patch:
diff --git a/.github/workflows/nigthly-tests.yaml b/.github/workflows/nigthly-tests.yaml
new file mode 100644
index 00000000..c2981721
--- /dev/null
+++ b/.github/workflows/nigthly-tests.yaml
@@ -0,0 +1,292 @@
+name: Nightly tests
+
+on:
+  schedule:
+    - cron: "0 6 * * *" # 6am UTC, daily.
+  workflow_dispatch:
+
+jobs:
+
+  linux:
+    name: linux
+    strategy:
+      matrix:
+        go: [ '1.22', '1.23' ]
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Set up Go
+        uses: actions/setup-go@v4
+        with:
+          go-version: ${{ matrix.go }}
+
+      - run: GOTRACEBACK=all make test testobjiotracing generate
+
+      - name: Assert workspace clean
+        run: scripts/check-workspace-clean.sh
+
+      - name: Post issue on failure
+        if: failure()
+        id: create-or-update-unique-issue
+        uses: ./.github/actions/post-issue
+        with:
+          title: "nightly test failed"
+          body: "The nightly ${{ github.job_id }} test run failed on ${{ github.sha }}. Please review the run [here](${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }})."
+          unique-title-includes: "nightly test failed"
+          labels: "C-test-failure"
+
+
+  linux-32bit:
+    name: linux-32bit
+    strategy:
+      matrix:
+        go: [ '1.22', '1.23' ]
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Set up Go
+        uses: actions/setup-go@v4
+        with:
+          go-version: ${{ matrix.go }}
+
+      - run: GOTRACEBACK=all GOARCH=386 make test
+
+      - name: Post issue on failure
+        if: failure()
+        id: create-or-update-unique-issue
+        uses: ./.github/actions/post-issue
+        with:
+          title: "nightly test failed"
+          body: "The nightly ${{ github.job_id }} test run failed on ${{ github.sha }}. Please review the run [here](${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }})."
+          unique-title-includes: "nightly test failed"
+          labels: "C-test-failure"
+
+
+  linux-arm:
+    name: linux-arm
+    strategy:
+      matrix:
+        go: [ '1.22', '1.23' ]
+    runs-on: ubuntu-22.04-arm
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Set up Go
+        uses: actions/setup-go@v4
+        with:
+          go-version: ${{ matrix.go }}
+
+      - run: GOTRACEBACK=all make test
+
+      - name: Post issue on failure
+        if: failure()
+        id: create-or-update-unique-issue
+        uses: ./.github/actions/post-issue
+        with:
+          title: "nightly test failed"
+          body: "The nightly ${{ github.job_id }} test run failed on ${{ github.sha }}. Please review the run [here](${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }})."
+          unique-title-includes: "nightly test failed"
+          labels: "C-test-failure"
+
+
+  linux-crossversion:
+    name: linux-crossversion
+    strategy:
+      matrix:
+        go: [ '1.22', '1.23' ]
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          # This target needs to find the crl-release-* branches.
+          fetch-depth: 0
+
+      - name: Set up Go
+        uses: actions/setup-go@v4
+        with:
+          go-version: ${{ matrix.go }}
+
+      - run: make crossversion-meta
+
+      - name: Post issue on failure
+        if: failure()
+        id: create-or-update-unique-issue
+        uses: ./.github/actions/post-issue
+        with:
+          title: "nightly test failed"
+          body: "The nightly ${{ github.job_id }} test run failed on ${{ github.sha }}. Please review the run [here](${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }})."
+          unique-title-includes: "nightly test failed"
+          labels: "C-test-failure"
+
+
+  linux-no-invariants:
+    name: linux-no-invariants
+    strategy:
+      matrix:
+        go: [ '1.22', '1.23' ]
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Set up Go
+        uses: actions/setup-go@v4
+        with:
+          go-version: ${{ matrix.go }}
+
+      - run: GOTRACEBACK=all make test TAGS=
+
+      - name: Post issue on failure
+        if: failure()
+        id: create-or-update-unique-issue
+        uses: ./.github/actions/post-issue
+        with:
+          title: "nightly test failed"
+          body: "The nightly ${{ github.job_id }} test run failed on ${{ github.sha }}. Please review the run [here](${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }})."
+          unique-title-includes: "nightly test failed"
+          labels: "C-test-failure"
+
+
+  linux-no-cgo:
+    name: linux-no-cgo
+    strategy:
+      matrix:
+        go: [ '1.22', '1.23' ]
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Set up Go
+        uses: actions/setup-go@v4
+        with:
+          go-version: ${{ matrix.go }}
+
+      - run: GOTRACEBACK=all CGO_ENABLED=0 make test TAGS=
+
+      - name: Post issue on failure
+        if: failure()
+        id: create-or-update-unique-issue
+        uses: ./.github/actions/post-issue
+        with:
+          title: "nightly test failed"
+          body: "The nightly ${{ github.job_id }} test run failed on ${{ github.sha }}. Please review the run [here](${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }})."
+          unique-title-includes: "nightly test failed"
+          labels: "C-test-failure"
+
+
+  darwin:
+    name: macos
+    strategy:
+      matrix:
+        go: [ '1.22', '1.23' ]
+    runs-on: macos-14
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Set up Go
+        uses: actions/setup-go@v4
+        with:
+          go-version: ${{ matrix.go }}
+
+      - run: GOTRACEBACK=all make test
+
+      - name: Post issue on failure
+        if: failure()
+        id: create-or-update-unique-issue
+        uses: ./.github/actions/post-issue
+        with:
+          title: "nightly test failed"
+          body: "The nightly ${{ github.job_id }} test run failed on ${{ github.sha }}. Please review the run [here](${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }})."
+          unique-title-includes: "nightly test failed"
+          labels: "C-test-failure"
+
+
+  windows:
+    name: windows
+    strategy:
+      matrix:
+        go: [ '1.22', '1.23' ]
+    runs-on: windows-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Set up Go
+        uses: actions/setup-go@v4
+        with:
+          go-version: ${{ matrix.go }}
+
+      - run: go test -tags invariants ./...
+
+      - name: Post issue on failure
+        if: failure()
+        id: create-or-update-unique-issue
+        uses: ./.github/actions/post-issue
+        with:
+          title: "nightly test failed"
+          body: "The nightly ${{ github.job_id }} test run failed on ${{ github.sha }}. Please review the run [here](${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }})."
+          unique-title-includes: "nightly test failed"
+          labels: "C-test-failure"
+
+  lint-checks:
+    name: lint-checks
+    strategy:
+      matrix:
+        go: [ '1.22', '1.23' ]
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Set up Go
+        uses: actions/setup-go@v4
+        with:
+          go-version: ${{ matrix.go }}
+
+      - name: mod-tidy-check
+        run: make mod-tidy-check
+
+      - name: format-check
+        run: make format-check
+
+      - name: Post issue on failure
+        if: failure()
+        id: create-or-update-unique-issue
+        uses: ./.github/actions/post-issue
+        with:
+          title: "nightly test failed"
+          body: "The nightly ${{ github.job_id }} test run failed on ${{ github.sha }}. Please review the run [here](${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }})."
+          unique-title-includes: "nightly test failed"
+          labels: "C-test-failure"
+
+
+  # Builds for other architectures.
+  build-other:
+    name: build-other
+    strategy:
+      matrix:
+        go: [ '1.22', '1.23' ]
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Set up Go
+        uses: actions/setup-go@v4
+        with:
+          go-version: ${{ matrix.go }}
+
+      - run: GOARCH=mips go build ./...
+      - run: GOARCH=mipsle go build ./...
+      - run: GOARCH=mips64le go build ./...
+      - run: GOOS=freebsd go build -v ./...
+      - run: GOOS=netbsd go build -v ./...
+      - run: GOOS=openbsd go build -v ./...
+
+      - name: Post issue on failure
+        if: failure()
+        id: create-or-update-unique-issue
+        uses: ./.github/actions/post-issue
+        with:
+          title: "nightly test failed"
+          body: "The nightly ${{ github.job_id }} test run failed on ${{ github.sha }}. Please review the run [here](${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }})."
+          unique-title-includes: "nightly test failed"
+          labels: "C-test-failure"