Repository: cockroachdb/cockroach
Pull request #143405: drt: scripts and YAML for upgrade-test
Author: login: nameisbhaskar
Labels: name: v25.2.0-prerelease
Created: 2025-03-25T04:35:31Z | Merged: 2025-04-09T12:42:22Z

This PR adds the scripts and YAML for running the upgrade testing.

Epic: None
Release: None


Diff:
diff --git a/pkg/cmd/drtprod/configs/drt_upgrade.yaml b/pkg/cmd/drtprod/configs/drt_upgrade.yaml
new file mode 100644
index 000000000000..3cabc7b6a6a3
--- /dev/null
+++ b/pkg/cmd/drtprod/configs/drt_upgrade.yaml
@@ -0,0 +1,163 @@
+# Yaml for creating and configuring the drt-upgrade-test and workload-upgrade-test clusters. This also configures the datadog.
+environment:
+  ROACHPROD_GCE_DEFAULT_SERVICE_ACCOUNT: 622274581499-compute@developer.gserviceaccount.com
+  ROACHPROD_DNS: drt.crdb.io
+  ROACHPROD_GCE_DNS_DOMAIN: drt.crdb.io
+  ROACHPROD_GCE_DNS_ZONE: drt
+  ROACHPROD_GCE_DEFAULT_PROJECT: cockroach-drt
+  CLUSTER: drt-upgrade-test
+  CLUSTER_NODES: 6
+  WORKLOAD_CLUSTER: workload-upgrade-test
+  WORKLOAD_NODES: 1
+  COCKROACH_VERSION: v24.3.8
+
+targets:
+  - target_name: $CLUSTER
+    steps:
+      - command: create
+        args:
+          - $CLUSTER
+        flags:
+          clouds: gce
+          gce-managed: true
+          gce-enable-multiple-stores: true
+          gce-zones: "us-east1-c"
+          nodes: $CLUSTER_NODES
+          gce-machine-type: n2-standard-8
+          os-volume-size: 100
+          local-ssd: true
+          gce-local-ssd-count: 4
+          username: drt
+          lifetime: 8760h
+          gce-image: "ubuntu-2204-jammy-v20240319"
+        on_rollback:
+          - command: destroy
+            args:
+              - $CLUSTER
+      - command: sync
+        flags:
+          clouds: gce
+      - command: stage
+        args:
+          - $CLUSTER
+          - release
+          - $COCKROACH_VERSION
+      - script: "pkg/cmd/drtprod/scripts/setup_datadog_cluster"
+      - command: start
+        args:
+          - $CLUSTER
+          - "--binary"
+          - "./cockroach"
+        flags:
+          enable-fluent-sink: true
+          store-count: 4
+          args: --wal-failover=among-stores
+          restart: false
+          sql-port: 26257
+        on_rollback:
+          - command: stop
+            args:
+              - $CLUSTER
+      - command: run
+        args:
+          - $CLUSTER
+          - --
+          - "sudo systemctl unmask cron.service ; sudo systemctl enable cron.service ; echo \"crontab -l ; echo '@reboot sleep 100 && ~/cockroach.sh' | crontab -\" > t.sh ; sh t.sh ; rm t.sh"
+  - target_name: $WORKLOAD_CLUSTER
+    steps:
+      - command: create
+        args:
+          - $WORKLOAD_CLUSTER
+        flags:
+          clouds: gce
+          gce-zones: "us-east1-c"
+          nodes: $WORKLOAD_NODES
+          gce-machine-type: n2-standard-4
+          os-volume-size: 100
+          username: workload
+          lifetime: 8760h
+        on_rollback:
+          - command: destroy
+            args:
+              - $WORKLOAD_CLUSTER
+      - command: sync
+        flags:
+          clouds: gce
+      - command: stage
+        args:
+          - $WORKLOAD_CLUSTER
+          - cockroach
+      - script: "pkg/cmd/drtprod/scripts/setup_datadog_workload"
+      - command: put
+        args:
+          - $WORKLOAD_CLUSTER
+          - artifacts/drtprod
+      - command: put
+        args:
+          - $WORKLOAD_CLUSTER
+          - artifacts/roachtest
+          - roachtest-operations
+      - command: put
+        args:
+          - $WORKLOAD_CLUSTER
+          - pkg/cmd/drtprod/scripts/mixed_version.sh
+      - command: run
+        args:
+          - $WORKLOAD_CLUSTER
+          - --
+          - "sudo systemctl unmask cron.service ; sudo systemctl enable cron.service ; sudo systemctl start cron.service ; echo \"crontab -l ; echo '0 4 * * * /home/ubuntu/mixed_version.sh $CLUSTER >> /home/ubuntu/mixed_version.log' | crontab -\" > t.sh ; sh t.sh ; rm t.sh"
+  - target_name: post_tasks
+    dependent_targets:
+      - $CLUSTER
+      - $WORKLOAD_CLUSTER
+    steps:
+      - script: pkg/cmd/drtprod/scripts/populate_workload_keys.sh
+      - script: rm
+        args:
+          - -rf
+          - certs-$CLUSTER
+      - command: get
+        args:
+          - $CLUSTER:1
+          - certs
+          - certs-$CLUSTER
+      - command: ssh
+        args:
+          - $WORKLOAD_CLUSTER
+          - --
+          - sudo
+          - rm
+          - -rf
+          - certs
+      - command: put
+        args:
+          - $WORKLOAD_CLUSTER
+          - certs-$CLUSTER
+          - certs
+      - command: ssh
+        args:
+          - $WORKLOAD_CLUSTER
+          - --
+          - chmod
+          - 600
+          - './certs/*'
+      - script: "pkg/cmd/drtprod/scripts/tpcc_init.sh"
+        args:
+          - cct_tpcc # suffix added to script name tpcc_init_cct_tpcc.sh
+          - false # determines whether to execute the script immediately on workload node
+        flags:
+          warehouses: 12000
+          db: cct_tpcc
+      - script: "pkg/cmd/drtprod/scripts/generate_tpcc_run.sh"
+        args:
+          - cct_tpcc # suffix added to script name tpcc_run.sh
+          - true # determines whether to execute the script immediately on workload node
+        flags:
+          db: cct_tpcc
+          warehouses: 12000
+          max-rate: 500
+          workers: 50
+          conns: 50
+          duration: 12h
+          ramp: 10m
+          wait: 0
diff --git a/pkg/cmd/drtprod/scripts/generate_tpcc_run.sh b/pkg/cmd/drtprod/scripts/generate_tpcc_run.sh
index a791370cfa46..ec369e32ae77 100755
--- a/pkg/cmd/drtprod/scripts/generate_tpcc_run.sh
+++ b/pkg/cmd/drtprod/scripts/generate_tpcc_run.sh
@@ -100,6 +100,7 @@ for ((NODE=0; NODE<WORKLOAD_NODES; NODE++)); do
 #!/usr/bin/env bash
 
 ./drtprod sync
+$([ "$execute_script" = "true" ] && [ "$NODE" -eq 0 ] && echo "${pwd}/tpcc_init_${suffix}.sh")
 PGURLS=\$(./drtprod pgurl $CLUSTER | sed s/\'//g)
 read -r -a PGURLS_ARR <<< "\$PGURLS"
 j=0
diff --git a/pkg/cmd/drtprod/scripts/mixed_version.sh b/pkg/cmd/drtprod/scripts/mixed_version.sh
new file mode 100755
index 000000000000..65d976cac548
--- /dev/null
+++ b/pkg/cmd/drtprod/scripts/mixed_version.sh
@@ -0,0 +1,123 @@
+#!/bin/bash
+
+# Copyright 2025 The Cockroach Authors.
+#
+# Use of this software is governed by the CockroachDB Software License
+# included in the /LICENSE file.
+
+# This script schedules daily maintenance commands on a 2‐week cycle.
+# Optional parameters OLD_RELEASE and NEW_RELEASE can be provided.
+# Usage: ./mixed_version.sh <CLUSTER> [OLD_RELEASE=v24.3.8] [NEW_RELEASE=v25.2.0-alpha.2]
+
+# Schedule of commands:
+#
+# Week 1:
+# - Tuesday:
+#     - Wipe the cluster
+#     - Full cluster restart with OLD_RELEASE
+#     - Set cluster.preserve_downgrade_option to OLD_RELEASE
+#     - upgrade nodes 1-3 to NEW_RELEASE (50% nodes)
+#     - Start tpcc_init_cct_tpcc.sh and roachtest_operations_run.sh
+# - Thursday:
+#     - Deploy NEW_RELEASE to the entire cluster (not finalized)
+# - Friday:
+#     - Revert all nodes to OLD_RELEASE
+#     - upgrade nodes 1-2 to NEW_RELEASE (33% nodes)
+#
+# Week 2:
+# - Monday:
+#     - Upgrade nodes 3-5 to NEW_RELEASE (80% nodes as 1,2 were already upgraded)
+# - Friday:
+#     - Reset cluster.preserve_downgrade_option
+#     - Upgrade node 6 to NEW_RELEASE (100% nodes as 1-5 were already upgraded)
+
+CLUSTER="$1"
+if [ -z "$CLUSTER" ]; then
+    echo "Usage: $0 <CLUSTER> [OLD_RELEASE] [NEW_RELEASE]"
+    exit 1
+fi
+
+/home/ubuntu/drtprod sync
+
+# Set optional release versions
+OLD_RELEASE="${2:-v24.3.8}"
+NEW_RELEASE="${3:-v25.2.0-alpha.2}"
+
+# Get today's day of week (1 for Monday, ... 7 for Sunday) and date
+day_of_week=$(date +%u)
+today=$(date +%F)
+cycle_file="/home/ubuntu/.cycle_info.txt"
+
+if [ -f "$cycle_file" ]; then
+    read saved_cycle saved_day < "$cycle_file"
+else
+    # Initialize cycle_week to 0 if no previous info exists
+    saved_cycle=0
+    saved_day=$today
+fi
+
+# On Monday, if this is the first run of today, flip the cycle week
+if [ "$day_of_week" -eq 1 ] && [ "$saved_day" != "$today" ]; then
+    cycle_week=$((1 - saved_cycle))
+else
+    cycle_week=$saved_cycle
+fi
+
+# Save the cycle week and today's date for persistence
+echo "$cycle_week $today" > "$cycle_file"
+
+# Use an array to store multiple commands
+cmds=()
+
+if [ "$day_of_week" -eq 1 ] && [ "$cycle_week" -eq 1 ]; then
+        # Week 2 - Monday
+        cmds+=("/home/ubuntu/drtprod deploy $CLUSTER:3-5 release $NEW_RELEASE")
+elif [ "$day_of_week" -eq 2 ] && [ "$cycle_week" -eq 0 ]; then
+        # Tuesday in Week 1 only
+        cmds+=("sudo systemctl stop tpcc_run_cct_tpcc")
+        cmds+=("sudo systemctl stop roachtest_ops")
+        cmds+=("/home/ubuntu/drtprod stop $CLUSTER")
+        cmds+=("/home/ubuntu/drtprod wipe $CLUSTER")
+        cmds+=("/home/ubuntu/drtprod stage $CLUSTER release $OLD_RELEASE")
+        cmds+=("/home/ubuntu/drtprod start $CLUSTER --binary ./cockroach --args=--wal-failover=among-stores --enable-fluent-sink=true --restart=false --sql-port=26257 --store-count=4")
+        version=$(echo "$OLD_RELEASE" | sed -E 's/^v([0-9]+\.[0-9]+)\..*/\1/')
+        cmds+=("/home/ubuntu/drtprod sql $CLUSTER:1 -- -e \"SET CLUSTER SETTING cluster.preserve_downgrade_option ='$version'\"")
+        cmds+=("/home/ubuntu/drtprod deploy $CLUSTER:1-3 release $NEW_RELEASE")
+        cmds+=("rm -rf /home/ubuntu/certs")
+        cmds+=("/home/ubuntu/drtprod get $CLUSTER:1 certs /home/ubuntu/certs")
+        cmds+=("chmod 600 /home/ubuntu/certs/*")
+        cmds+=("/home/ubuntu/tpcc_init_cct_tpcc.sh")
+        cmds+=("sudo systemd-run --unit tpcc_run_cct_tpcc --same-dir --uid $(id -u) --gid $(id -g) bash /home/ubuntu/tpcc_run_cct_tpcc.sh")
+        cmds+=("sleep 30")
+        # Note that roachtest_operations_run.sh needs to be setup manually for the first time.
+        cmds+=("sudo systemd-run --unit roachtest_ops --same-dir --uid $(id -u) --gid $(id -g) bash /home/ubuntu/roachtest_operations_run.sh")
+elif [ "$day_of_week" -eq 4 ] && [ "$cycle_week" -eq 0 ]; then
+    # Thursday in Week 1 only
+    cmds+=("/home/ubuntu/drtprod deploy $CLUSTER:4-6 release $NEW_RELEASE")
+elif [ "$day_of_week" -eq 5 ]; then
+    # Friday for both Weeks
+    if [ "$cycle_week" -eq 0 ]; then
+        # Week 1 friday commands
+        cmds+=("/home/ubuntu/drtprod deploy $CLUSTER release $OLD_RELEASE")
+        cmds+=("/home/ubuntu/drtprod deploy $CLUSTER:1-2 release $NEW_RELEASE")
+    else
+        # Week 2 friday commands
+        cmds+=("/home/ubuntu/drtprod sql $CLUSTER:1 -- -e 'RESET CLUSTER SETTING cluster.preserve_downgrade_option'")
+        cmds+=("/home/ubuntu/drtprod deploy $CLUSTER:6 release $NEW_RELEASE")
+    fi
+fi
+
+# Always check the status of the cluster
+cmds+=("/home/ubuntu/drtprod status $CLUSTER")
+
+if [ ${#cmds[@]} -gt 0 ]; then
+    for cmd in "${cmds[@]}"; do
+        echo "Executing: $cmd"
+        if ! eval "$cmd"; then
+            echo "Error executing: $cmd" >&2
+            exit 1
+        fi
+    done
+else
+    echo "No scheduled command for today."
+fi
diff --git a/pkg/cmd/drtprod/scripts/populate_workload_keys.sh b/pkg/cmd/drtprod/scripts/populate_workload_keys.sh
new file mode 100755
index 000000000000..3f2a3a74803c
--- /dev/null
+++ b/pkg/cmd/drtprod/scripts/populate_workload_keys.sh
@@ -0,0 +1,20 @@
+#!/bin/bash
+
+# Copyright 2025 The Cockroach Authors.
+#
+# Use of this software is governed by the CockroachDB Software License
+# included in the /LICENSE file.
+
+if [ -z "${CLUSTER}" ]; then
+  echo "environment CLUSTER is not set"
+  exit 1
+fi
+
+if [ -z "${WORKLOAD_CLUSTER}" ]; then
+  echo "environment CLUSTER is not set"
+  exit 1
+fi
+
+# the ssh keys of all workload nodes should be setup on the crdb nodes for the operations
+drtprod ssh ${CLUSTER} -- "echo \"$(drtprod run ${WORKLOAD_CLUSTER} -- cat ./.ssh/id_rsa.pub|grep ssh-rsa)\" >> ./.ssh/authorized_keys"
+


Reviews:
user: login: csgourav
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-30T17:45:13Z
user: login: vidit-bhat
state: COMMENTED
author_association: NONE
submitted_at: 2025-03-31T04:19:09Z
user: login: nameisbhaskar
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-04-03T18:32:37Z
user: login: nameisbhaskar
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-04-03T18:33:47Z
user: login: nameisbhaskar
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-04-03T18:34:25Z
user: login: vidit-bhat
state: COMMENTED
author_association: NONE
submitted_at: 2025-04-04T05:15:08Z
user: login: vidit-bhat
body: LGTM save one comment for staging `drtprod` binary on `CLUSTER` as well
state: APPROVED
author_association: NONE
submitted_at: 2025-04-09T09:21:49Z
user: login: nameisbhaskar
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-04-09T10:31:12Z
user: login: nameisbhaskar
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-04-09T10:32:17Z
user: login: csgourav
state: APPROVED
author_association: CONTRIBUTOR
submitted_at: 2025-04-09T10:35:20Z

Review comments:
diff_hunk: @@ -0,0 +1,162 @@
+# Yaml for creating and configuring the drt-chaos and workload-chaos clusters. This also configures the datadog.
+environment:
+  ROACHPROD_GCE_DEFAULT_SERVICE_ACCOUNT: 622274581499-compute@developer.gserviceaccount.com
+  ROACHPROD_DNS: drt.crdb.io
+  ROACHPROD_GCE_DNS_DOMAIN: drt.crdb.io
+  ROACHPROD_GCE_DNS_ZONE: drt
+  ROACHPROD_GCE_DEFAULT_PROJECT: cockroach-drt
+  CLUSTER: drt-upgrade-test
+  CLUSTER_NODES: 6
+  WORKLOAD_CLUSTER: workload-upgrade-test
+  WORKLOAD_NODES: 1
+  COCKROACH_VERSION: v24.3.8
+
+targets:
+  - target_name: $CLUSTER
+    steps:
+      - command: create
+        args:
+          - $CLUSTER
+        flags:
+          clouds: gce
+          gce-managed: true
+          gce-enable-multiple-stores: true
+          gce-zones: "us-east1-c"
+          nodes: $CLUSTER_NODES
+          gce-machine-type: n2-standard-8
+          local-ssd: true
+          gce-local-ssd-count: 4
+          username: drt
+          lifetime: 8760h
+          gce-image: "ubuntu-2204-jammy-v20240319"
+        on_rollback:
+          - command: destroy
+            args:
+              - $CLUSTER
+      - command: sync
+        flags:
+          clouds: gce
+      - command: stage
+        args:
+          - $CLUSTER
+          - release
+          - $COCKROACH_VERSION
+      - script: "pkg/cmd/drtprod/scripts/setup_datadog_cluster"
+      - command: start
+        args:
+          - $CLUSTER
+          - "--binary"
+          - "./cockroach"
+        flags:
+          enable-fluent-sink: true
+          store-count: 4
+          args: --wal-failover=among-stores
+          restart: false
+          sql-port: 26257
+        on_rollback:
+          - command: stop
+            args:
+              - $CLUSTER
+      - command: run
+        args:
+          - $CLUSTER
+          - --
+          - "sudo systemctl unmask cron.service ; sudo systemctl enable cron.service ; echo \"crontab -l ; echo '@reboot sleep 100 && ~/cockroach.sh' | crontab -\" > t.sh ; sh t.sh ; rm t.sh"
+  - target_name: $WORKLOAD_CLUSTER
+    steps:
+      - command: create
+        args:
+          - $WORKLOAD_CLUSTER
+        flags:
+          clouds: gce
+          gce-zones: "us-east1-c"
+          nodes: $WORKLOAD_NODES
+          gce-machine-type: n2-standard-4
+          os-volume-size: 100
path: pkg/cmd/drtprod/configs/drt_upgrade.yaml
user: login: csgourav
body: os-volume-size: 100 - should this be added to cockroach cluster as well?
created_at: 2025-03-30T17:36:20Z
author_association: CONTRIBUTOR
diff_hunk: @@ -0,0 +1,20 @@
+#!/bin/bash
+
+# Copyright 2025 The Cockroach Authors.
+#
+# Use of this software is governed by the CockroachDB Software License
+# included in the /LICENSE file.
+
+if [ -z "${CLUSTER}" ]; then
+  echo "environment CLUSTER is not set"
+  exit 1
+fi
+
+if [ -z "${WORKLOAD_CLUSTER}" ]; then
+  echo "environment CLUSTER is not set"
+  exit 1
+fi
+
+# the ssh keys of all workload nodes should be setup on the crdb nodes for the operations
+drtprod ssh ${CLUSTER} -- "echo \"$(drtprod run ${WORKLOAD_CLUSTER} -- cat ./.ssh/id_rsa.pub|grep ssh-rsa)\" >> ./.ssh/authorized_keys"
path: pkg/cmd/drtprod/scripts/populate_workload_keys.sh
user: login: csgourav
body: this will need a drtprod binary on ${CLUSTER}, but it seems we haven't copied this yet
created_at: 2025-03-30T17:42:20Z
author_association: CONTRIBUTOR
diff_hunk: @@ -0,0 +1,162 @@
+# Yaml for creating and configuring the drt-chaos and workload-chaos clusters. This also configures the datadog.
path: pkg/cmd/drtprod/configs/drt_upgrade.yaml
user: login: csgourav
body: nit: update cluster names in comment
created_at: 2025-03-30T17:45:00Z
author_association: CONTRIBUTOR
diff_hunk: @@ -0,0 +1,162 @@
+# Yaml for creating and configuring the drt-chaos and workload-chaos clusters. This also configures the datadog.
+environment:
+  ROACHPROD_GCE_DEFAULT_SERVICE_ACCOUNT: 622274581499-compute@developer.gserviceaccount.com
+  ROACHPROD_DNS: drt.crdb.io
+  ROACHPROD_GCE_DNS_DOMAIN: drt.crdb.io
+  ROACHPROD_GCE_DNS_ZONE: drt
+  ROACHPROD_GCE_DEFAULT_PROJECT: cockroach-drt
+  CLUSTER: drt-upgrade-test
+  CLUSTER_NODES: 6
+  WORKLOAD_CLUSTER: workload-upgrade-test
+  WORKLOAD_NODES: 1
+  COCKROACH_VERSION: v24.3.8
+
+targets:
+  - target_name: $CLUSTER
+    steps:
+      - command: create
+        args:
+          - $CLUSTER
+        flags:
+          clouds: gce
+          gce-managed: true
+          gce-enable-multiple-stores: true
+          gce-zones: "us-east1-c"
+          nodes: $CLUSTER_NODES
+          gce-machine-type: n2-standard-8
+          local-ssd: true
+          gce-local-ssd-count: 4
+          username: drt
+          lifetime: 8760h
+          gce-image: "ubuntu-2204-jammy-v20240319"
+        on_rollback:
+          - command: destroy
+            args:
+              - $CLUSTER
+      - command: sync
+        flags:
+          clouds: gce
path: pkg/cmd/drtprod/configs/drt_upgrade.yaml
user: login: csgourav
body: is this flag needed for sync command?
created_at: 2025-03-30T17:45:03Z
author_association: CONTRIBUTOR
diff_hunk: @@ -0,0 +1,120 @@
+#!/bin/bash
+
+# Copyright 2025 The Cockroach Authors.
+#
+# Use of this software is governed by the CockroachDB Software License
+# included in the /LICENSE file.
+
+# This script schedules daily maintenance commands on a 2‐week cycle.
+# Optional parameters OLD_RELEASE and NEW_RELEASE can be provided.
+# Usage: ./mixed_version.sh <CLUSTER> [OLD_RELEASE=v24.3.8] [NEW_RELEASE=v25.2.0-alpha.1]
+
+# Schedule of commands:
+#
+# Week 1:
+# - Tuesday:
+#     - Wipe the cluster
+#     - Full cluster restart with OLD_RELEASE
+#     - Set cluster.preserve_downgrade_option to OLD_RELEASE
+#     - upgrade nodes 1-3 to NEW_RELEASE (50% nodes)
+#     - Start tpcc_init_cct_tpcc.sh and roachtest_operations_run.sh
+# - Thursday:
+#     - Deploy NEW_RELEASE to the entire cluster (not finalized)
+# - Friday:
+#     - Revert all nodes to OLD_RELEASE
+#     - upgrade nodes 1-2 to NEW_RELEASE (33% nodes)
+#
+# Week 2:
+# - Monday:
+#     - Upgrade nodes 3-5 to NEW_RELEASE (80% nodes as 1,2 were already upgraded)
+# - Friday:
+#     - Reset cluster.preserve_downgrade_option
+#     - Upgrade node 6 to NEW_RELEASE (100% nodes as 1-5 were already upgraded)
+
+CLUSTER="$1"
+if [ -z "$CLUSTER" ]; then
+    echo "Usage: $0 <CLUSTER> [OLD_RELEASE] [NEW_RELEASE]"
+    exit 1
+fi
+
+/home/ubuntu/drtprod sync
+
+# Set optional release versions
+OLD_RELEASE="${2:-v24.3.8}"
+NEW_RELEASE="${3:-v25.2.0-alpha.1}"
+
+# Get today's day of week (1 for Monday, ... 7 for Sunday) and date
+day_of_week=$(date +%u)
+today=$(date +%F)
+cycle_file="/home/ubuntu/.cycle_info.txt"
+
+if [ -f "$cycle_file" ]; then
+    read saved_cycle saved_day < "$cycle_file"
+else
+    # Initialize cycle_week to 0 if no previous info exists
+    saved_cycle=0
+    saved_day=$today
+fi
+
+# On Monday, if this is the first run of today, flip the cycle week
+if [ "$day_of_week" -eq 1 ] && [ "$saved_day" != "$today" ]; then
+    cycle_week=$((1 - saved_cycle))
+else
+    cycle_week=$saved_cycle
+fi
+
+# Save the cycle week and today's date for persistence
+echo "$cycle_week $today" > "$cycle_file"
+
+# Use an array to store multiple commands
+cmds=()
+
+if [ "$day_of_week" -eq 1 ] && [ "$cycle_week" -eq 1 ]; then
+        # Week 2 - Monday
+        cmds+=("/home/ubuntu/drtprod deploy $CLUSTER:3-5 release $NEW_RELEASE")
+elif [ "$day_of_week" -eq 2 ] && [ "$cycle_week" -eq 0 ]; then
+        # Tuesday in Week 1 only
+        cmds+=("sudo systemctl stop tpcc_run_cct_tpcc")
+        cmds+=("sudo systemctl stop roachtest_ops")
+        cmds+=("/home/ubuntu/drtprod stop $CLUSTER")
+        cmds+=("/home/ubuntu/drtprod wipe $CLUSTER")
+        cmds+=("/home/ubuntu/drtprod stage $CLUSTER release $OLD_RELEASE")
+        cmds+=("/home/ubuntu/drtprod start $CLUSTER --binary ./cockroach --args=--wal-failover=among-stores --enable-fluent-sink=true --restart=false --sql-port=26257 --store-count=4")
+        version=$(echo "$OLD_RELEASE" | sed -E 's/^v([0-9]+\.[0-9]+)\..*/\1/')
+        cmds+=("/home/ubuntu/drtprod sql $CLUSTER:1 -- -e \"SET CLUSTER SETTING cluster.preserve_downgrade_option ='$version'\"")
+        cmds+=("/home/ubuntu/drtprod deploy $CLUSTER:1-3 release $NEW_RELEASE")
+        cmds+=("rm -rf /home/ubuntu/certs")
+        cmds+=("/home/ubuntu/drtprod get $CLUSTER:1 certs /home/ubuntu/certs")
+        cmds+=("chmod 600 /home/ubuntu/certs/*")
+        cmds+=("/home/ubuntu/tpcc_init_cct_tpcc.sh")
+        cmds+=("sudo systemd-run --unit tpcc_run_cct_tpcc --same-dir --uid $(id -u) --gid $(id -g) bash /home/ubuntu/tpcc_run_cct_tpcc.sh")
+        cmds+=("sleep 30")
+        # Note that roachtest_operations_run.sh needs to be setup manually for the first time.
+        cmds+=("sudo systemd-run --unit roachtest_ops --same-dir --uid $(id -u) --gid $(id -g) bash /home/ubuntu/roachtest_operations_run.sh")
+elif [ "$day_of_week" -eq 4 ] && [ "$cycle_week" -eq 0 ]; then
+    # Thursday in Week 1 only
+    cmds+=("/home/ubuntu/drtprod deploy $CLUSTER:4-6 release $NEW_RELEASE")
+elif [ "$day_of_week" -eq 5 ]; then
+    # Friday for both Weeks
+    if [ "$cycle_week" -eq 0 ]; then
+        # Week 1 friday commands
+        cmds+=("/home/ubuntu/drtprod deploy $CLUSTER release $OLD_RELEASE")
+        cmds+=("/home/ubuntu/drtprod deploy $CLUSTER:1-2 release $NEW_RELEASE")
+    else
+        # Week 2 friday commands
+        cmds+=("/home/ubuntu/drtprod sql $CLUSTER:1 -- -e 'RESET CLUSTER SETTING cluster.preserve_downgrade_option'")
+        cmds+=("/home/ubuntu/drtprod deploy $CLUSTER:6 release $NEW_RELEASE")
+    fi
+fi
+
+# Always check the status of the cluster
+cmds+=("/home/ubuntu/drtprod status $CLUSTER")
+
+if [ ${#cmds[@]} -gt 0 ]; then
+    for cmd in "${cmds[@]}"; do
+        echo "Executing: $cmd"
+        eval "$cmd"
path: pkg/cmd/drtprod/scripts/mixed_version.sh
user: login: vidit-bhat
body: Should we exit in case of critical errors? Or are we okay to continue silently?
```
if ! eval "$cmd"; then
    echo "Error executing: $cmd" >&2
    exit 1
fi
```

created_at: 2025-03-31T04:19:09Z
author_association: NONE
diff_hunk: @@ -0,0 +1,20 @@
+#!/bin/bash
+
+# Copyright 2025 The Cockroach Authors.
+#
+# Use of this software is governed by the CockroachDB Software License
+# included in the /LICENSE file.
+
+if [ -z "${CLUSTER}" ]; then
+  echo "environment CLUSTER is not set"
+  exit 1
+fi
+
+if [ -z "${WORKLOAD_CLUSTER}" ]; then
+  echo "environment CLUSTER is not set"
+  exit 1
+fi
+
+# the ssh keys of all workload nodes should be setup on the crdb nodes for the operations
+drtprod ssh ${CLUSTER} -- "echo \"$(drtprod run ${WORKLOAD_CLUSTER} -- cat ./.ssh/id_rsa.pub|grep ssh-rsa)\" >> ./.ssh/authorized_keys"
path: pkg/cmd/drtprod/scripts/populate_workload_keys.sh
user: login: nameisbhaskar
body: We do copy drtprod
```
      - command: put
        args:
          - $WORKLOAD_CLUSTER
          - artifacts/drtprod
```
created_at: 2025-04-03T18:32:37Z
author_association: CONTRIBUTOR
diff_hunk: @@ -0,0 +1,162 @@
+# Yaml for creating and configuring the drt-chaos and workload-chaos clusters. This also configures the datadog.
+environment:
+  ROACHPROD_GCE_DEFAULT_SERVICE_ACCOUNT: 622274581499-compute@developer.gserviceaccount.com
+  ROACHPROD_DNS: drt.crdb.io
+  ROACHPROD_GCE_DNS_DOMAIN: drt.crdb.io
+  ROACHPROD_GCE_DNS_ZONE: drt
+  ROACHPROD_GCE_DEFAULT_PROJECT: cockroach-drt
+  CLUSTER: drt-upgrade-test
+  CLUSTER_NODES: 6
+  WORKLOAD_CLUSTER: workload-upgrade-test
+  WORKLOAD_NODES: 1
+  COCKROACH_VERSION: v24.3.8
+
+targets:
+  - target_name: $CLUSTER
+    steps:
+      - command: create
+        args:
+          - $CLUSTER
+        flags:
+          clouds: gce
+          gce-managed: true
+          gce-enable-multiple-stores: true
+          gce-zones: "us-east1-c"
+          nodes: $CLUSTER_NODES
+          gce-machine-type: n2-standard-8
+          local-ssd: true
+          gce-local-ssd-count: 4
+          username: drt
+          lifetime: 8760h
+          gce-image: "ubuntu-2204-jammy-v20240319"
+        on_rollback:
+          - command: destroy
+            args:
+              - $CLUSTER
+      - command: sync
+        flags:
+          clouds: gce
path: pkg/cmd/drtprod/configs/drt_upgrade.yaml
user: login: nameisbhaskar
body: Yes, this is to ensure that we do not go into the AWS cloud. In AWS cloud it reads all clusters of roachprod as it does not understand the `cockroach-drt` project.
created_at: 2025-04-03T18:33:47Z
author_association: CONTRIBUTOR
diff_hunk: @@ -0,0 +1,120 @@
+#!/bin/bash
+
+# Copyright 2025 The Cockroach Authors.
+#
+# Use of this software is governed by the CockroachDB Software License
+# included in the /LICENSE file.
+
+# This script schedules daily maintenance commands on a 2‐week cycle.
+# Optional parameters OLD_RELEASE and NEW_RELEASE can be provided.
+# Usage: ./mixed_version.sh <CLUSTER> [OLD_RELEASE=v24.3.8] [NEW_RELEASE=v25.2.0-alpha.1]
+
+# Schedule of commands:
+#
+# Week 1:
+# - Tuesday:
+#     - Wipe the cluster
+#     - Full cluster restart with OLD_RELEASE
+#     - Set cluster.preserve_downgrade_option to OLD_RELEASE
+#     - upgrade nodes 1-3 to NEW_RELEASE (50% nodes)
+#     - Start tpcc_init_cct_tpcc.sh and roachtest_operations_run.sh
+# - Thursday:
+#     - Deploy NEW_RELEASE to the entire cluster (not finalized)
+# - Friday:
+#     - Revert all nodes to OLD_RELEASE
+#     - upgrade nodes 1-2 to NEW_RELEASE (33% nodes)
+#
+# Week 2:
+# - Monday:
+#     - Upgrade nodes 3-5 to NEW_RELEASE (80% nodes as 1,2 were already upgraded)
+# - Friday:
+#     - Reset cluster.preserve_downgrade_option
+#     - Upgrade node 6 to NEW_RELEASE (100% nodes as 1-5 were already upgraded)
+
+CLUSTER="$1"
+if [ -z "$CLUSTER" ]; then
+    echo "Usage: $0 <CLUSTER> [OLD_RELEASE] [NEW_RELEASE]"
+    exit 1
+fi
+
+/home/ubuntu/drtprod sync
+
+# Set optional release versions
+OLD_RELEASE="${2:-v24.3.8}"
+NEW_RELEASE="${3:-v25.2.0-alpha.1}"
+
+# Get today's day of week (1 for Monday, ... 7 for Sunday) and date
+day_of_week=$(date +%u)
+today=$(date +%F)
+cycle_file="/home/ubuntu/.cycle_info.txt"
+
+if [ -f "$cycle_file" ]; then
+    read saved_cycle saved_day < "$cycle_file"
+else
+    # Initialize cycle_week to 0 if no previous info exists
+    saved_cycle=0
+    saved_day=$today
+fi
+
+# On Monday, if this is the first run of today, flip the cycle week
+if [ "$day_of_week" -eq 1 ] && [ "$saved_day" != "$today" ]; then
+    cycle_week=$((1 - saved_cycle))
+else
+    cycle_week=$saved_cycle
+fi
+
+# Save the cycle week and today's date for persistence
+echo "$cycle_week $today" > "$cycle_file"
+
+# Use an array to store multiple commands
+cmds=()
+
+if [ "$day_of_week" -eq 1 ] && [ "$cycle_week" -eq 1 ]; then
+        # Week 2 - Monday
+        cmds+=("/home/ubuntu/drtprod deploy $CLUSTER:3-5 release $NEW_RELEASE")
+elif [ "$day_of_week" -eq 2 ] && [ "$cycle_week" -eq 0 ]; then
+        # Tuesday in Week 1 only
+        cmds+=("sudo systemctl stop tpcc_run_cct_tpcc")
+        cmds+=("sudo systemctl stop roachtest_ops")
+        cmds+=("/home/ubuntu/drtprod stop $CLUSTER")
+        cmds+=("/home/ubuntu/drtprod wipe $CLUSTER")
+        cmds+=("/home/ubuntu/drtprod stage $CLUSTER release $OLD_RELEASE")
+        cmds+=("/home/ubuntu/drtprod start $CLUSTER --binary ./cockroach --args=--wal-failover=among-stores --enable-fluent-sink=true --restart=false --sql-port=26257 --store-count=4")
+        version=$(echo "$OLD_RELEASE" | sed -E 's/^v([0-9]+\.[0-9]+)\..*/\1/')
+        cmds+=("/home/ubuntu/drtprod sql $CLUSTER:1 -- -e \"SET CLUSTER SETTING cluster.preserve_downgrade_option ='$version'\"")
+        cmds+=("/home/ubuntu/drtprod deploy $CLUSTER:1-3 release $NEW_RELEASE")
+        cmds+=("rm -rf /home/ubuntu/certs")
+        cmds+=("/home/ubuntu/drtprod get $CLUSTER:1 certs /home/ubuntu/certs")
+        cmds+=("chmod 600 /home/ubuntu/certs/*")
+        cmds+=("/home/ubuntu/tpcc_init_cct_tpcc.sh")
+        cmds+=("sudo systemd-run --unit tpcc_run_cct_tpcc --same-dir --uid $(id -u) --gid $(id -g) bash /home/ubuntu/tpcc_run_cct_tpcc.sh")
+        cmds+=("sleep 30")
+        # Note that roachtest_operations_run.sh needs to be setup manually for the first time.
+        cmds+=("sudo systemd-run --unit roachtest_ops --same-dir --uid $(id -u) --gid $(id -g) bash /home/ubuntu/roachtest_operations_run.sh")
+elif [ "$day_of_week" -eq 4 ] && [ "$cycle_week" -eq 0 ]; then
+    # Thursday in Week 1 only
+    cmds+=("/home/ubuntu/drtprod deploy $CLUSTER:4-6 release $NEW_RELEASE")
+elif [ "$day_of_week" -eq 5 ]; then
+    # Friday for both Weeks
+    if [ "$cycle_week" -eq 0 ]; then
+        # Week 1 friday commands
+        cmds+=("/home/ubuntu/drtprod deploy $CLUSTER release $OLD_RELEASE")
+        cmds+=("/home/ubuntu/drtprod deploy $CLUSTER:1-2 release $NEW_RELEASE")
+    else
+        # Week 2 friday commands
+        cmds+=("/home/ubuntu/drtprod sql $CLUSTER:1 -- -e 'RESET CLUSTER SETTING cluster.preserve_downgrade_option'")
+        cmds+=("/home/ubuntu/drtprod deploy $CLUSTER:6 release $NEW_RELEASE")
+    fi
+fi
+
+# Always check the status of the cluster
+cmds+=("/home/ubuntu/drtprod status $CLUSTER")
+
+if [ ${#cmds[@]} -gt 0 ]; then
+    for cmd in "${cmds[@]}"; do
+        echo "Executing: $cmd"
+        eval "$cmd"
path: pkg/cmd/drtprod/scripts/mixed_version.sh
user: login: nameisbhaskar
body: Yes, the thought was to bypass any error. I can add this.
created_at: 2025-04-03T18:34:24Z
author_association: CONTRIBUTOR
diff_hunk: @@ -0,0 +1,20 @@
+#!/bin/bash
+
+# Copyright 2025 The Cockroach Authors.
+#
+# Use of this software is governed by the CockroachDB Software License
+# included in the /LICENSE file.
+
+if [ -z "${CLUSTER}" ]; then
+  echo "environment CLUSTER is not set"
+  exit 1
+fi
+
+if [ -z "${WORKLOAD_CLUSTER}" ]; then
+  echo "environment CLUSTER is not set"
+  exit 1
+fi
+
+# the ssh keys of all workload nodes should be setup on the crdb nodes for the operations
+drtprod ssh ${CLUSTER} -- "echo \"$(drtprod run ${WORKLOAD_CLUSTER} -- cat ./.ssh/id_rsa.pub|grep ssh-rsa)\" >> ./.ssh/authorized_keys"
path: pkg/cmd/drtprod/scripts/populate_workload_keys.sh
user: login: vidit-bhat
body: But we're running the `drtprod` command in `$CLUSTER`. We would need it there as well.
created_at: 2025-04-04T05:15:08Z
author_association: NONE
diff_hunk: @@ -0,0 +1,20 @@
+#!/bin/bash
+
+# Copyright 2025 The Cockroach Authors.
+#
+# Use of this software is governed by the CockroachDB Software License
+# included in the /LICENSE file.
+
+if [ -z "${CLUSTER}" ]; then
+  echo "environment CLUSTER is not set"
+  exit 1
+fi
+
+if [ -z "${WORKLOAD_CLUSTER}" ]; then
+  echo "environment CLUSTER is not set"
+  exit 1
+fi
+
+# the ssh keys of all workload nodes should be setup on the crdb nodes for the operations
+drtprod ssh ${CLUSTER} -- "echo \"$(drtprod run ${WORKLOAD_CLUSTER} -- cat ./.ssh/id_rsa.pub|grep ssh-rsa)\" >> ./.ssh/authorized_keys"
path: pkg/cmd/drtprod/scripts/populate_workload_keys.sh
user: login: nameisbhaskar
body: Yes, that is why it is put in the worklaod node. We execute it from the workload node.
created_at: 2025-04-09T10:31:12Z
author_association: CONTRIBUTOR
diff_hunk: @@ -0,0 +1,20 @@
+#!/bin/bash
+
+# Copyright 2025 The Cockroach Authors.
+#
+# Use of this software is governed by the CockroachDB Software License
+# included in the /LICENSE file.
+
+if [ -z "${CLUSTER}" ]; then
+  echo "environment CLUSTER is not set"
+  exit 1
+fi
+
+if [ -z "${WORKLOAD_CLUSTER}" ]; then
+  echo "environment CLUSTER is not set"
+  exit 1
+fi
+
+# the ssh keys of all workload nodes should be setup on the crdb nodes for the operations
+drtprod ssh ${CLUSTER} -- "echo \"$(drtprod run ${WORKLOAD_CLUSTER} -- cat ./.ssh/id_rsa.pub|grep ssh-rsa)\" >> ./.ssh/authorized_keys"
path: pkg/cmd/drtprod/scripts/populate_workload_keys.sh
user: login: nameisbhaskar
body: $(drtprod run ${WORKLOAD_CLUSTER} -- cat ./.ssh/id_rsa.pub|grep ssh-rsa) gets executed in the local machione itself and only the value is passed to the actual cluster.
created_at: 2025-04-09T10:32:17Z
author_association: CONTRIBUTOR

PR comments:
user: login: cockroach-teamcity
created_at: 2025-03-25T04:35:42Z
body: 

<!-- Reviewable:start -->
This change is [<img src="https://reviewable.io/review_button.svg" height="34" align="absmiddle" alt="Reviewable"/>](https://reviewable.io/reviews/cockroachdb/cockroach/143405)
<!-- Reviewable:end -->

author_association: MEMBER
user: login: nameisbhaskar
created_at: 2025-04-03T18:39:50Z
body: Thanks @csgourav and @vidit-bhat for the review.  I have updated the code. PTAL.
author_association: CONTRIBUTOR
user: login: nameisbhaskar
created_at: 2025-04-09T10:37:07Z
body: > LGTM save one comment for staging `drtprod` binary on `CLUSTER` as well

As explain above, the "drtprod run ${WORKLOAD_CLUSTER}" is actually run locally itself. So, we do not need the binary in the cluster. Thanks for the approval. I will go ahead and merge it!
author_association: CONTRIBUTOR
user: login: nameisbhaskar
created_at: 2025-04-09T10:37:20Z
body: bors r=@vidit-bhat,@csgourav
author_association: CONTRIBUTOR
user: login: craig[bot]
created_at: 2025-04-09T11:41:38Z
body: This PR was included in a batch that successfully built, but then failed to merge into master (it was a non-fast-forward update). It will be automatically retried.
author_association: CONTRIBUTOR
user: login: craig[bot]
created_at: 2025-04-09T12:42:22Z
body: Build succeeded:
  * [local_roachtest](https://github.com/cockroachdb/cockroach/actions/runs/14356814456/job/40248094650)
  * [examples_orms](https://github.com/cockroachdb/cockroach/actions/runs/14356814456/job/40248092549)
  * [unit_tests](https://github.com/cockroachdb/cockroach/actions/runs/14356814456/job/40248096708)
  * [docker_image_amd64](https://github.com/cockroachdb/cockroach/actions/runs/14356814456/job/40248091801)
  * [linux_amd64_fips_build](https://github.com/cockroachdb/cockroach/actions/runs/14356814456/job/40248096150)
  * [linux_amd64_build](https://github.com/cockroachdb/cockroach/actions/runs/14356814456/job/40248093998)
  * [check_generated_code](https://github.com/cockroachdb/cockroach/actions/runs/14356814456/job/40248091190)
  * [acceptance](https://github.com/cockroachdb/cockroach/actions/runs/14356814456/job/40248090260)
  * [local_roachtest_fips](https://github.com/cockroachdb/cockroach/actions/runs/14356814456/job/40248095610)
  * [lint](https://github.com/cockroachdb/cockroach/actions/runs/14356814456/job/40248093331)
author_association: CONTRIBUTOR