Repository: cockroachdb/cockroach
Pull request #142942: drt: YAML for 300 node scale cluster
Author: login: nameisbhaskar
Labels: name: v25.2.0-prerelease
Created: 2025-03-15T02:21:26Z | Merged: 2025-03-16T03:02:16Z

This PR includes the changes in the YAML for running the 300 node scale cluster There are 2 YAML files:
1. create cluster with 150 node
2. grow cluster to next 150 nodes after running drtprod grow

Epic: None
Release: None


Diff:
diff --git a/pkg/cmd/drtprod/configs/drt_scale_300.yaml b/pkg/cmd/drtprod/configs/drt_scale_300.yaml
index e3ba3c51f2fb..83cde92a9136 100644
--- a/pkg/cmd/drtprod/configs/drt_scale_300.yaml
+++ b/pkg/cmd/drtprod/configs/drt_scale_300.yaml
@@ -8,8 +8,19 @@ environment:
   ROACHPROD_GCE_DEFAULT_PROJECT: cockroach-drt
   CLUSTER: drt-scale-300
   WORKLOAD_CLUSTER: workload-scale-300
-  CLUSTER_NODES: 300
-  WORKLOAD_NODES: 20
+  CLUSTER_NODES: 150
+  TOTAL_PARTITIONS: 30
+  WORKLOAD_NODES: 15
+
+dependent_file_locations:
+  - pkg/cmd/drtprod/scripts/setup_datadog_cluster
+  - pkg/cmd/drtprod/scripts/setup_datadog_workload
+  - pkg/cmd/drtprod/scripts/ycsb_init.sh
+  - pkg/cmd/drtprod/scripts/generate_ycsb_run.sh
+  - pkg/cmd/drtprod/scripts/tpcc_init.sh
+  - pkg/cmd/drtprod/scripts/generate_tpcc_run.sh
+  - pkg/cmd/drtprod/scripts/ycsb_init.sh
+  - pkg/cmd/drtprod/scripts/generate_ycsb_run.sh
 
 targets:
   # crdb cluster specs
@@ -22,11 +33,11 @@ targets:
           clouds: gce
           gce-managed: true
           gce-enable-multiple-stores: true
-          gce-zones: "us-central1-a:150,us-central1-b:150"
+          gce-zones: "us-central1-a:50,us-central1-b:50,us-central1-c:50"
           nodes: $CLUSTER_NODES
           gce-machine-type: n2-standard-16
           local-ssd: false
-          gce-pd-volume-size: 375
+          gce-pd-volume-size: 1024
           gce-pd-volume-type: pd-ssd
           gce-pd-volume-count: 4
           os-volume-size: 100
@@ -39,7 +50,8 @@ targets:
       - command: stage
         args:
           - $CLUSTER
-          - cockroach
+          - release
+          - v25.1.1
       - script: "pkg/cmd/drtprod/scripts/setup_datadog_cluster"
       - command: start
         args:
@@ -58,6 +70,12 @@ targets:
           - $CLUSTER
           - --
           - "sudo systemctl unmask cron.service ; sudo systemctl enable cron.service ; echo \"crontab -l ; echo '@reboot sleep 100 && ~/cockroach.sh' | crontab -\" > t.sh ; sh t.sh ; rm t.sh"
+      - command: sql
+        args:
+          - $CLUSTER:1
+          - --
+          - -e
+          - "SET CLUSTER SETTING kv.snapshot_rebalance.max_rate='256 MB'"
   # workload cluster specs
   - target_name: $WORKLOAD_CLUSTER
     steps:
@@ -66,7 +84,7 @@ targets:
           - $WORKLOAD_CLUSTER
         flags:
           clouds: gce
-          gce-zones: "us-central1-a,us-central1-b"
+          gce-zones: "us-central1-f"
           nodes: $WORKLOAD_NODES
           gce-machine-type: n2-standard-8
           os-volume-size: 100
@@ -84,10 +102,6 @@ targets:
         args:
           - $WORKLOAD_CLUSTER
           - cockroach
-      - command: stage
-        args:
-          - $WORKLOAD_CLUSTER
-          - workload
       - script: "pkg/cmd/drtprod/scripts/setup_datadog_workload"
   - target_name: post_tasks
     dependent_targets:
@@ -118,13 +132,34 @@ targets:
       - command: put
         args:
           - $WORKLOAD_CLUSTER
-          - artifacts/roachprod
-          - roachprod
-      - command: put
+          - artifacts/drtprod
+          - drtprod
+      - script: "pkg/cmd/drtprod/scripts/tpcc_init.sh"
         args:
-          - $WORKLOAD_CLUSTER
-          - artifacts/roachtest
-          - roachtest-operations
+          - cct_tpcc_6M
+          - false
+        flags:
+          warehouses: 6000000
+          db: cct_tpcc_6M
+  - target_name: tpcc_run
+    dependent_targets:
+      - $CLUSTER
+      - $WORKLOAD_CLUSTER
+    steps:
+      - script: "pkg/cmd/drtprod/scripts/generate_tpcc_run.sh"
+        args:
+          - cct_tpcc_6M # suffix added to script name tpcc_run_400k.sh
+          - false # determines whether to execute the script immediately on workload node
+        flags:
+          db: cct_tpcc_6M
+          warehouses: 6000000
+          active-warehouses: 500000
+          conns: 33333
+          max-rate: 2500
+          workers: 166666
+          duration: 12h
+          ramp: 1h
+          wait: 0
       - script: "pkg/cmd/drtprod/scripts/ycsb_init.sh"
         args:
           - 20M
diff --git a/pkg/cmd/drtprod/configs/drt_scale_300_grow.yaml b/pkg/cmd/drtprod/configs/drt_scale_300_grow.yaml
new file mode 100644
index 000000000000..a3de30b96bae
--- /dev/null
+++ b/pkg/cmd/drtprod/configs/drt_scale_300_grow.yaml
@@ -0,0 +1,36 @@
+# Yaml for creating and configuring the drt-scale cluster. This also configures Datadog.
+# Build the roachprod and roachtest binaries (using --cross) before running this script
+environment:
+  ROACHPROD_GCE_DEFAULT_SERVICE_ACCOUNT: 622274581499-compute@developer.gserviceaccount.com
+  ROACHPROD_DNS: drt.crdb.io
+  ROACHPROD_GCE_DNS_DOMAIN: drt.crdb.io
+  ROACHPROD_GCE_DNS_ZONE: drt
+  ROACHPROD_GCE_DEFAULT_PROJECT: cockroach-drt
+  CLUSTER: drt-scale-300:151-300
+
+targets:
+  - target_name: $CLUSTER
+    steps:
+      - command: stage
+        args:
+          - $CLUSTER
+          - release
+          - v25.1.1
+      - script: "pkg/cmd/drtprod/scripts/setup_datadog_cluster"
+      - command: start
+        args:
+          - $CLUSTER
+          - "--binary"
+          - "./cockroach"
+        flags:
+          # add flag to set provisioned throughput on each store according to their cloud provider limits
+          enable-fluent-sink: true
+          store-count: 4
+          args: --wal-failover=among-stores
+          restart: false
+          sql-port: 26257
+      - command: run
+        args:
+          - $CLUSTER
+          - --
+          - "sudo systemctl unmask cron.service ; sudo systemctl enable cron.service ; echo \"crontab -l ; echo '@reboot sleep 100 && ~/cockroach.sh' | crontab -\" > t.sh ; sh t.sh ; rm t.sh"
diff --git a/pkg/cmd/drtprod/scripts/generate_tpcc_run.sh b/pkg/cmd/drtprod/scripts/generate_tpcc_run.sh
index 38ad7de21ea3..a791370cfa46 100755
--- a/pkg/cmd/drtprod/scripts/generate_tpcc_run.sh
+++ b/pkg/cmd/drtprod/scripts/generate_tpcc_run.sh
@@ -5,9 +5,10 @@
 # Use of this software is governed by the CockroachDB Software License
 # included in the /LICENSE file.
 
-# This script sets up the tpcc run workload script in the workload nodes
-# The --warehouses flag is passed as argument to this script
-# NOTE - This uses CLUSTER and WORKLOAD_CLUSTER environment variable, if not set the script fails
+# Sets up TPCC workload script across nodes with optional client side data partitioning.
+# Usage: <script_suffix> <execute:true|false> [workload flags]
+# Requires: CLUSTER, WORKLOAD_CLUSTER, WORKLOAD_NODES env vars
+# Optional: TOTAL_PARTITIONS (defaults to WORKLOAD_NODES if WORKLOAD_NODES>1)
 if [ "$#" -lt 2 ]; then
   echo "Usage: $0 <script_suffix> <execute:true|false> <flags to init:--warehouses,--db>"
   exit 1
@@ -39,31 +40,67 @@ if [ -z "${WORKLOAD_NODES}" ]; then
   exit 1
 fi
 
-if [ -z "${CLUSTER_NODES}" ]; then
-  echo "environment CLUSTER_NODES is not set"
+# Set TOTAL_PARTITIONS equal to WORKLOAD_NODES if not set
+if [ -z "${TOTAL_PARTITIONS}" ] && [ "${WORKLOAD_NODES}" -gt 1 ]; then
+  TOTAL_PARTITIONS=$WORKLOAD_NODES
+  echo "TOTAL_PARTITIONS not set and WORKLOAD_NODES > 1, defaulting to WORKLOAD_NODES ($WORKLOAD_NODES)"
+fi
+
+# Ensure TOTAL_PARTITIONS is not less than WORKLOAD_NODES
+if [ -n "${TOTAL_PARTITIONS}" ] && [ "${TOTAL_PARTITIONS}" -lt "${WORKLOAD_NODES}" ]; then
+  echo "TOTAL_PARTITIONS ($TOTAL_PARTITIONS) must be greater than or equal to WORKLOAD_NODES ($WORKLOAD_NODES)"
   exit 1
 fi
 
+get_partitions_in_range() {
+    local start=$(($1 - 1))
+    local end=$(($2 - 1))
+
+    if [[ -z "$start" || -z "$end" || "$start" -lt 0 || "$end" -ge $TOTAL_PARTITIONS || "$start" -gt "$end" ]]; then
+        echo "Invalid range. Provide values between 0 and $((TOTAL_PARTITIONS - 1)), with start <= end."
+        return 1
+    fi
+
+    local result=""
+    for (( i=start; i<=end; i++ )); do
+        if [ -z "$result" ]; then
+            result="$i"
+        else
+            result="${result},$i"
+        fi
+    done
+
+    echo "$result"
+}
+
 absolute_path=$(drtprod run "${WORKLOAD_CLUSTER}":1 -- "realpath ./cockroach")
 pwd=$(drtprod run "${WORKLOAD_CLUSTER}":1 -- "dirname ${absolute_path}")
 
-# Calculate the number of PGURLS each workload node should get
-PGURL_PER_NODE=$((CLUSTER_NODES / WORKLOAD_NODES))
-REMAINDER_NODE=$((CLUSTER_NODES % WORKLOAD_NODES))
-
-# Distribute the PGURLS among the workload nodes
 for ((NODE=0; NODE<WORKLOAD_NODES; NODE++)); do
-  START_OFFSET=$((NODE * PGURL_PER_NODE + (NODE < REMAINDER_NODE ? NODE : REMAINDER_NODE) + 1))
-  END_OFFSET=$((START_OFFSET + PGURL_PER_NODE + (NODE < REMAINDER_NODE ? 1 : 0) - 1))
+  # Handle partitioning if TOTAL_PARTITIONS is set
+  partition_args=""
+  if [ -n "${TOTAL_PARTITIONS}" ]; then
+    PARTITION_PER_NODE=$((TOTAL_PARTITIONS / WORKLOAD_NODES))
+    PARTITION_REMAINDER_NODE=$((TOTAL_PARTITIONS % WORKLOAD_NODES))
 
-  # Print or use the PGURLS for the current workload node
-  echo "pgurl for Nodes ${START_OFFSET}:${END_OFFSET}"
+    PARTITION_START_OFFSET=$((NODE * PARTITION_PER_NODE + (NODE < PARTITION_REMAINDER_NODE ? NODE : PARTITION_REMAINDER_NODE) + 1))
+    PARTITION_END_OFFSET=$((PARTITION_START_OFFSET + PARTITION_PER_NODE + (NODE < PARTITION_REMAINDER_NODE ? 1 : 0) - 1))
+
+    parts=$(get_partitions_in_range ${PARTITION_START_OFFSET} ${PARTITION_END_OFFSET})
+    if [ $? -ne 0 ]; then
+        echo "Failed to get partitions in range"
+        exit 1
+    fi
+    echo "Partition assignment: $parts"
+    partition_args="--client-partitions=$TOTAL_PARTITIONS --partition-affinity=$parts"
+  fi
 
   # Create the workload script
   cat <<EOF >/tmp/tpcc_run_${suffix}.sh
 #!/usr/bin/env bash
 
-PGURLS=\$(./roachprod pgurl $CLUSTER:$START_OFFSET-$END_OFFSET | sed s/\'//g)
+./drtprod sync
+PGURLS=\$(./drtprod pgurl $CLUSTER | sed s/\'//g)
 read -r -a PGURLS_ARR <<< "\$PGURLS"
 j=0
 while true; do
@@ -71,6 +108,7 @@ while true; do
     ((j++))
     LOG=./tpcc_\$j.txt
     ./cockroach workload run tpcc $@ \
+        $partition_args \
         --tolerate-errors \
         --families \
          \${PGURLS_ARR[@]}  | tee \$LOG
@@ -81,12 +119,12 @@ while true; do
 done
 EOF
 
-#   Upload the script to the workload cluster
+  # Upload the script to the workload cluster
   drtprod put $WORKLOAD_CLUSTER:$((NODE + 1)) /tmp/tpcc_run_${suffix}.sh
   drtprod ssh $WORKLOAD_CLUSTER:$((NODE + 1)) -- "chmod +x tpcc_run_${suffix}.sh"
   if [ "$execute_script" = "true" ]; then
       drtprod run "${WORKLOAD_CLUSTER}":$((NODE + 1)) -- "sudo systemd-run --unit tpcc_run_${suffix} --same-dir --uid \$(id -u) --gid \$(id -g) bash ${pwd}/tpcc_run_${suffix}.sh"
   else
     echo "Run --> drtprod run "${WORKLOAD_CLUSTER}":$((NODE + 1)) -- \"sudo systemd-run --unit tpcc_run_${suffix} --same-dir --uid \\\$(id -u) --gid \\\$(id -g) bash ${pwd}/tpcc_run_${suffix}.sh\""
-    fi
+  fi
 done
diff --git a/pkg/cmd/drtprod/scripts/ycsb_init.sh b/pkg/cmd/drtprod/scripts/ycsb_init.sh
index 7e55be4c6276..d5b0149844c4 100755
--- a/pkg/cmd/drtprod/scripts/ycsb_init.sh
+++ b/pkg/cmd/drtprod/scripts/ycsb_init.sh
@@ -44,7 +44,7 @@ PGURLS=$(drtprod pgurl "${CLUSTER}":1)
 drtprod ssh "${WORKLOAD_CLUSTER}":1 -- "tee ycsb_init_${suffix}.sh > /dev/null << 'EOF'
 #!/bin/bash
 
-${pwd}/cockroach workload init ycsb $PGURLS --drop --families=false $@
+${pwd}/cockroach workload init ycsb $PGURLS --drop --families=false --insert-hash=false $@
 EOF"
 drtprod ssh "${WORKLOAD_CLUSTER}":1 -- "chmod +x ycsb_init_${suffix}.sh"
 


Reviews:
user: login: vidit-bhat
state: COMMENTED
author_association: NONE
submitted_at: 2025-03-15T03:00:32Z
user: login: shailendra-patel
state: CHANGES_REQUESTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-15T04:39:19Z
user: login: shailendra-patel
state: APPROVED
author_association: CONTRIBUTOR
submitted_at: 2025-03-15T11:38:37Z

Review comments:
diff_hunk: @@ -44,33 +44,68 @@ if [ -z "${CLUSTER_NODES}" ]; then
   exit 1
 fi
 
+get_partitions_in_range() {
+    local start=$(($1 - 1))
+    local end=$(($2 - 1))
+
+    if [[ -z "$start" || -z "$end" || "$start" -lt 0 || "$end" -ge $WORKLOAD_NODES || "$start" -gt "$end" ]]; then
+        echo "Invalid range. Provide values between 0 and $((WORKLOAD_NODES - 1)), with start <= end."
+        return 1
+    fi
+
+    local result=""
+    for (( i=start; i<=end; i++ )); do
+        if [ -z "$result" ]; then
+            result="$i"
+        else
+            result="${result},$i"
+        fi
+    done
+
+    echo "$result"
+}
+
 absolute_path=$(drtprod run "${WORKLOAD_CLUSTER}":1 -- "realpath ./cockroach")
 pwd=$(drtprod run "${WORKLOAD_CLUSTER}":1 -- "dirname ${absolute_path}")
 
 # Calculate the number of PGURLS each workload node should get
 PGURL_PER_NODE=$((CLUSTER_NODES / WORKLOAD_NODES))
 REMAINDER_NODE=$((CLUSTER_NODES % WORKLOAD_NODES))
+PARTITION_PER_NODE=$((WORKLOAD_NODES / WORKLOAD_NODES))
+PARTITION_REMAINDER_NODE=$((WORKLOAD_NODES % WORKLOAD_NODES))
path: pkg/cmd/drtprod/scripts/generate_tpcc_run.sh
user: login: vidit-bhat
body: Are these correct?
created_at: 2025-03-15T03:00:32Z
author_association: NONE
diff_hunk: @@ -44,33 +44,68 @@ if [ -z "${CLUSTER_NODES}" ]; then
   exit 1
 fi
 
+get_partitions_in_range() {
path: pkg/cmd/drtprod/scripts/generate_tpcc_run.sh
user: login: shailendra-patel
body: Attaching the modified `generate_tpcc_run.sh` script.
```shell
#!/bin/bash

# Copyright 2024 The Cockroach Authors.
#
# Use of this software is governed by the CockroachDB Software License
# included in the /LICENSE file.

# Sets up TPCC workload script across nodes with optional client side data partitioning.
# Usage: <script_suffix> <execute:true|false> [workload flags]
# Requires: CLUSTER, WORKLOAD_CLUSTER, WORKLOAD_NODES env vars
# Optional: TOTAL_PARTITIONS (defaults to WORKLOAD_NODES if WORKLOAD_NODES>1)
if [ "$#" -lt 2 ]; then
  echo "Usage: $0 <script_suffix> <execute:true|false> <flags to init:--warehouses,--db>"
  exit 1
fi
suffix=$1
shift
# The second argument represents whether the init process should be started in the workload cluster
# The value is true or false
if [ "$1" != "true" ] && [ "$1" != "false" ]; then
  # $1 is used again because of the shift
  echo "Error: The second argument must be 'true' or 'false' which implies whether the script should be started in background or not."
  exit 1
fi
execute_script=$1
shift

if [ -z "${CLUSTER}" ]; then
  echo "environment CLUSTER is not set"
  exit 1
fi

if [ -z "${WORKLOAD_CLUSTER}" ]; then
  echo "environment WORKLOAD_CLUSTER is not set"
  exit 1
fi

if [ -z "${WORKLOAD_NODES}" ]; then
  echo "environment WORKLOAD_NODES is not set"
  exit 1
fi

# Set TOTAL_PARTITIONS equal to WORKLOAD_NODES if not set
if [ -z "${TOTAL_PARTITIONS}" ] && [ "${WORKLOAD_NODES}" -gt 1 ]; then
  TOTAL_PARTITIONS=$WORKLOAD_NODES
  echo "TOTAL_PARTITIONS not set and WORKLOAD_NODES > 1, defaulting to WORKLOAD_NODES ($WORKLOAD_NODES)"
fi

# Ensure TOTAL_PARTITIONS is not less than WORKLOAD_NODES
if [ -n "${TOTAL_PARTITIONS}" ] && [ "${TOTAL_PARTITIONS}" -lt "${WORKLOAD_NODES}" ]; then
  echo "TOTAL_PARTITIONS ($TOTAL_PARTITIONS) must be greater than or equal to WORKLOAD_NODES ($WORKLOAD_NODES)"
  exit 1
fi

get_partitions_in_range() {
    local start=$(($1 - 1))
    local end=$(($2 - 1))

    if [[ -z "$start" || -z "$end" || "$start" -lt 0 || "$end" -ge $TOTAL_PARTITIONS || "$start" -gt "$end" ]]; then
        echo "Invalid range. Provide values between 0 and $((TOTAL_PARTITIONS - 1)), with start <= end."
        return 1
    fi

    local result=""
    for (( i=start; i<=end; i++ )); do
        if [ -z "$result" ]; then
            result="$i"
        else
            result="${result},$i"
        fi
    done

    echo "$result"
}

absolute_path=$(drtprod run "${WORKLOAD_CLUSTER}":1 -- "realpath ./cockroach")
pwd=$(drtprod run "${WORKLOAD_CLUSTER}":1 -- "dirname ${absolute_path}")

for ((NODE=0; NODE<WORKLOAD_NODES; NODE++)); do
  # Handle partitioning if TOTAL_PARTITIONS is set
  partition_args=""
  if [ -n "${TOTAL_PARTITIONS}" ]; then
    PARTITION_PER_NODE=$((TOTAL_PARTITIONS / WORKLOAD_NODES))
    PARTITION_REMAINDER_NODE=$((TOTAL_PARTITIONS % WORKLOAD_NODES))

    PARTITION_START_OFFSET=$((NODE * PARTITION_PER_NODE + (NODE < PARTITION_REMAINDER_NODE ? NODE : PARTITION_REMAINDER_NODE) + 1))
    PARTITION_END_OFFSET=$((PARTITION_START_OFFSET + PARTITION_PER_NODE + (NODE < PARTITION_REMAINDER_NODE ? 1 : 0) - 1))

    parts=$(get_partitions_in_range ${PARTITION_START_OFFSET} ${PARTITION_END_OFFSET})
    if [ $? -ne 0 ]; then
        echo "Failed to get partitions in range"
        exit 1
    fi
    echo "Partition assignment: $parts"
    partition_args="--client-partitions=$TOTAL_PARTITIONS --partition-affinity=$parts"
  fi

  # Create the workload script
  cat <<EOF >/tmp/tpcc_run_${suffix}.sh
#!/usr/bin/env bash

./drtprod sync
PGURLS=\$(./drtprod pgurl $CLUSTER | sed s/\'//g)
read -r -a PGURLS_ARR <<< "\$PGURLS"
j=0
while true; do
    echo ">> Starting tpcc workload"
    ((j++))
    LOG=./tpcc_\$j.txt
    ./cockroach workload run tpcc $@ \
        $partition_args \
        --tolerate-errors \
        --families \
         \${PGURLS_ARR[@]}  | tee \$LOG
    if [ \$? -eq 0 ]; then
        rm "\$LOG"
    fi
    sleep 1
done
EOF

  # Upload the script to the workload cluster
  drtprod put $WORKLOAD_CLUSTER:$((NODE + 1)) /tmp/tpcc_run_${suffix}.sh
  drtprod ssh $WORKLOAD_CLUSTER:$((NODE + 1)) -- "chmod +x tpcc_run_${suffix}.sh"
  if [ "$execute_script" = "true" ]; then
      drtprod run "${WORKLOAD_CLUSTER}":$((NODE + 1)) -- "sudo systemd-run --unit tpcc_run_${suffix} --same-dir --uid \$(id -u) --gid \$(id -g) bash ${pwd}/tpcc_run_${suffix}.sh"
  else
    echo "Run --> drtprod run "${WORKLOAD_CLUSTER}":$((NODE + 1)) -- \"sudo systemd-run --unit tpcc_run_${suffix} --same-dir --uid \\\$(id -u) --gid \\\$(id -g) bash ${pwd}/tpcc_run_${suffix}.sh\""
  fi
done

```
created_at: 2025-03-15T04:24:54Z
author_association: CONTRIBUTOR
diff_hunk: @@ -8,8 +8,18 @@ environment:
   ROACHPROD_GCE_DEFAULT_PROJECT: cockroach-drt
   CLUSTER: drt-scale-300
   WORKLOAD_CLUSTER: workload-scale-300
-  CLUSTER_NODES: 300
-  WORKLOAD_NODES: 20
+  CLUSTER_NODES: 150
+  WORKLOAD_NODES: 10
path: pkg/cmd/drtprod/configs/drt_scale_300.yaml
user: login: shailendra-patel
body: Add environment variable `TOTAL_PARTITIONS`
created_at: 2025-03-15T04:27:32Z
author_association: CONTRIBUTOR
diff_hunk: @@ -118,13 +131,34 @@ targets:
       - command: put
         args:
           - $WORKLOAD_CLUSTER
-          - artifacts/roachprod
-          - roachprod
-      - command: put
+          - artifacts/drtprod
+          - drtprod
+      - script: "pkg/cmd/drtprod/scripts/tpcc_init.sh"
         args:
-          - $WORKLOAD_CLUSTER
-          - artifacts/roachtest
-          - roachtest-operations
+          - cct_tpcc_6M
+          - false
+        flags:
+          warehouses: 6000000
+          db: cct_tpcc_6M
+  - target_name: tpcc_run
+    dependent_targets:
+      - $CLUSTER
+      - $WORKLOAD_CLUSTER
+    steps:
+      - script: "pkg/cmd/drtprod/scripts/generate_tpcc_run.sh"
+        args:
+          - cct_tpcc_6M # suffix added to script name tpcc_run_400k.sh
+          - false # determines whether to execute the script immediately on workload node
+        flags:
+          db: cct_tpcc_6M
+          warehouses: 6000000
+          active-warehouses: 100000
+          conns: 20000
+          max-rate: 1000
+          workers: 100000
+          duration: 12h
+          ramp: 10m
+          wait: 0
path: pkg/cmd/drtprod/configs/drt_scale_300.yaml
user: login: shailendra-patel
body: The placeholder flag values are acceptable for now. For the actual run, we may run the following configuration: set `WORKLOAD_NODES=15`, `TOTAL_PARTITONS=30`, and use `500K` active warehouses. Additionally, each workload runner will be initiated at 10-minute intervals to manage load on crdb cluster. @BabuSrithar, please feel free to add more details.
```
db: cct_tpcc_6M
warehouses: 6000000
active-warehouses: 500000
conns: 33333
max-rate: 2500
workers: 166666
duration: 12h
ramp: 1h
wait: 0
```
created_at: 2025-03-15T04:38:41Z
author_association: CONTRIBUTOR

PR comments:
user: login: cockroach-teamcity
created_at: 2025-03-15T02:21:36Z
body: 

<!-- Reviewable:start -->
This change is [<img src="https://reviewable.io/review_button.svg" height="34" align="absmiddle" alt="Reviewable"/>](https://reviewable.io/reviews/cockroachdb/cockroach/142942)
<!-- Reviewable:end -->

author_association: MEMBER
user: login: nameisbhaskar
created_at: 2025-03-15T07:34:15Z
body: Thanks @shailendra-patel and @vidit-bhat for the review. The changes are done. PTAL.
I have not verified the script, but, reviewed it. Regarding the 10 mins interval, is it expected to be done manually?
author_association: CONTRIBUTOR
user: login: shailendra-patel
created_at: 2025-03-15T11:38:27Z
body: Thanks for accomodating the changes.
> I have not verified the script, but, reviewed it.

I have tested the script it works.

>  Regarding the 10 mins interval, is it expected to be done manually?

This can be done vai simple shell script like below. This shell script is not needed to be checked in.
```shell
#!/bin/bash

for i in {1..15}
do
  echo "$i"
  drtprod run workload-scale-300:${i} -- "sudo systemd-run --unit tpcc_run_cct_tpcc_2000k_cl_3m --same-dir --uid \$(id -u) --gid \$(id -g) bash /home/ubuntu/tpcc_run_cct_tpcc_2000k_cl_3m.sh"
  sleep 90
done
```

LGTM.

author_association: CONTRIBUTOR
user: login: nameisbhaskar
created_at: 2025-03-15T13:21:47Z
body:                                         

> Thanks for accomodating the changes.
> 
> > I have not verified the script, but, reviewed it.
> 
> I have tested the script it works.
> 
> > Regarding the 10 mins interval, is it expected to be done manually?
> 
> This can be done vai simple shell script like below. This shell script is not needed to be checked in.
> 
> ```shell
> #!/bin/bash
> 
> for i in {1..15}
> do
>   echo "$i"
>   drtprod run workload-scale-300:${i} -- "sudo systemd-run --unit tpcc_run_cct_tpcc_2000k_cl_3m --same-dir --uid \$(id -u) --gid \$(id -g) bash /home/ubuntu/tpcc_run_cct_tpcc_2000k_cl_3m.sh"
>   sleep 90
> done
> ```
> 
> LGTM.

Thanks! Yes, we can also add a sleep in each script and start all in one go so that nothing is blocking us in the execution side. simple `sleep $(( 90 * NODE ))`
author_association: CONTRIBUTOR
user: login: nameisbhaskar
created_at: 2025-03-15T15:21:13Z
body:  bors r=@shailendra-patel,@vidit-bhat 
author_association: CONTRIBUTOR
user: login: nameisbhaskar
created_at: 2025-03-16T02:33:07Z
body: bors r=@shailendra-patel,@vidit-bhat
author_association: CONTRIBUTOR
user: login: craig[bot]
created_at: 2025-03-16T03:02:13Z
body: Build succeeded:
  * [acceptance](https://github.com/cockroachdb/cockroach/actions/runs/13879049853/job/38835175187)
  * [check_generated_code](https://github.com/cockroachdb/cockroach/actions/runs/13879049853/job/38835174915)
  * [docker_image_amd64](https://github.com/cockroachdb/cockroach/actions/runs/13879049853/job/38835175049)
  * [examples_orms](https://github.com/cockroachdb/cockroach/actions/runs/13879049853/job/38835175427)
  * [lint](https://github.com/cockroachdb/cockroach/actions/runs/13879049853/job/38835175312)
  * [linux_amd64_build](https://github.com/cockroachdb/cockroach/actions/runs/13879049853/job/38835175729)
  * [linux_amd64_fips_build](https://github.com/cockroachdb/cockroach/actions/runs/13879049853/job/38835175841)
  * [local_roachtest](https://github.com/cockroachdb/cockroach/actions/runs/13879049853/job/38835175550)
  * [unit_tests](https://github.com/cockroachdb/cockroach/actions/runs/13879049853/job/38835175931)
  * [local_roachtest_fips](https://github.com/cockroachdb/cockroach/actions/runs/13879049853/job/38835175640)
author_association: CONTRIBUTOR