Add

2b855b11bc4/latest

@@ -0,0 +1 @@
+global_step52452

2b855b11bc4/sbatch_2b855b11bval.sh

@@ -0,0 +1,168 @@
+#!/bin/bash
+#SBATCH --exclude=nid007571,nid007112,nid006774,nid007502,nid007506,nid007507,nid005145,nid006692,nid007218,nid007123,nid006124,nid006123,nid007496,nid007237,nid006852,nid007206,nid006947,nid007212,nid006977,nid007222,nid005444,nid007219,nid007493,nid007221,nid005300,nid005619,nid006118,nid005203,nid006113,nid006481,nid007077,nid005208,nid005207,nid005879,nid005901
+#SBATCH --nodes=32
+#SBATCH --ntasks-per-node=1
+#SBATCH --cpus-per-task=32
+#SBATCH --mem=256G
+#SBATCH -p standard-g
+#SBATCH -t 48:00:00
+#SBATCH --gpus-per-node=mi250:8
+#SBATCH --exclusive=user
+#SBATCH --hint=nomultithread
+#SBATCH --account=project_462000119
+#SBATCH -o logs/%j.out
+#SBATCH -e logs/%j.err
+
+VARIANT=2b855b11bval
+VARIANT_CKPT=lm1-2b8-55b-c4-repetitions/2b855b11bc4
+
+# if run without sbatch, invoke here
+if [ -z $SLURM_JOB_ID ]; then
+    mkdir -p logs
+    sbatch "$0"
+    exit
+fi
+
+set -euo pipefail
+
+# symlink logs/latest.out and logs/latest.err
+ln -f -s $SLURM_JOB_ID.out logs/latest.out
+ln -f -s $SLURM_JOB_ID.err logs/latest.err
+
+KILL_SWITCH_PATH=kill-switch-$VARIANT
+CHECKPOINT_PATH=$VARIANT_CKPT
+TENSORBOARD_PATH=tensorboard_$VARIANT
+
+# Data
+VOCAB_FILE="gpt2/vocab.json"
+MERGE_FILE="gpt2/merges.txt"
+#DATA_PATH="/scratch/project_462000119/data/pile/megatron_data/meg-gpt2_pile_text_document"
+TRAIN_DATA_PATH=train1b5.txt
+# "train: 1.0 0:1 /scratch/project_462000119/data/c4_subsampled/gpt2tok_c4_en_12B_text_document"
+VALID_DATA_PATH=val.txt
+# "validation: 1.0 0:1 /scratch/project_462000119/data/c4_validation/gpt2tok_c4validation_rerun_text_document"
+
+PP_SIZE=1
+TP_SIZE=1
+
+MICRO_BATCH_SIZE=2
+GRADIENT_ACCUMULATION_STEPS=1
+WORLD_SIZE=$((SLURM_GPUS_ON_NODE*SLURM_JOB_NUM_NODES))
+GLOBAL_BATCH_SIZE=$((MICRO_BATCH_SIZE*WORLD_SIZE*GRADIENT_ACCUMULATION_STEPS))
+
+# Model parameters
+source model_params.sh
+MODEL_PARAM=("${PARAM_2980M[@]}")
+NHIDDEN=${MODEL_PARAM[0]}
+FFN_HIDDEN_SIZE=${MODEL_PARAM[1]}
+KV_SIZE=${MODEL_PARAM[2]}
+NHEADS=${MODEL_PARAM[3]}
+NLAYERS=${MODEL_PARAM[4]}
+SEQ_LEN=2048
+
+echo "Model parameters: d_model $NHIDDEN ffw_size $FFN_HIDDEN_SIZE kv_size $KV_SIZE n_heads $NHEADS n_layers $NLAYERS"
+
+SAVE_INTERVAL=1000
+
+# Tokens: 35546190000
+# -> Samples: 17356538
+TRAIN_SAMPLES=1
+
+OPTIMIZER_ARGS=" \
+    --optimizer adam \
+    --adam-beta1 0.9 \
+    --adam-beta2 0.999 \
+    --adam-eps 1e-8 \
+    --lr 2e-4 \
+    --min-lr 2e-5 \
+    --lr-decay-style cosine \
+    --lr-decay-samples $TRAIN_SAMPLES \
+    --lr-warmup-samples 0 \
+    --clip-grad 1.0 \
+    --weight-decay 1e-1 \
+    --override-lr-scheduler \
+    --reset-progress \
+    --no-load-optim \
+    "
+
+GPT_ARGS=" \
+    --num-layers $NLAYERS \
+    --hidden-size $NHIDDEN \
+    --num-attention-heads $NHEADS \
+    --kv-channels $KV_SIZE \
+    --ffn-hidden-size $FFN_HIDDEN_SIZE \
+    --seq-length $SEQ_LEN \
+    --max-position-embeddings $SEQ_LEN \
+    --micro-batch-size $MICRO_BATCH_SIZE \
+    --global-batch-size $GLOBAL_BATCH_SIZE \
+    --train-samples $TRAIN_SAMPLES \
+    --vocab-file $VOCAB_FILE \
+    --merge-file $MERGE_FILE \
+    --clip-grad 1.0 \
+    --kill-switch-path $KILL_SWITCH_PATH \
+    --bf16 \
+    $OPTIMIZER_ARGS \
+    "
+
+OUTPUT_ARGS=" \
+    --log-interval 10 \
+    --save-interval $SAVE_INTERVAL \
+    --eval-interval 1 \
+    --eval-iters 100 \
+    --eval-only true \
+    --tensorboard-dir $TENSORBOARD_PATH \
+    --tensorboard-queue-size 5 \
+    --log-timers-to-tensorboard \
+    --log-batch-size-to-tensorboard \
+    --log-validation-ppl-to-tensorboard \
+    "
+
+ZERO_STAGE=0
+
+mkdir -p ds_configs
+DS_CONFIG_PATH="ds_configs/$SLURM_JOB_ID.json"
+
+cat <<EOF > $DS_CONFIG_PATH
+{
+    "train_micro_batch_size_per_gpu": $MICRO_BATCH_SIZE,
+    "train_batch_size": $GLOBAL_BATCH_SIZE,
+    "gradient_clipping": 1.0,
+    "zero_optimization": {
+        "stage": $ZERO_STAGE
+    },
+    "bf16": {
+        "enabled": true
+    },
+    "steps_per_print": 2000,
+    "wall_clock_breakdown": false
+}
+EOF
+
+DEEPSPEED_ARGS=" \
+    --deepspeed \
+    --deepspeed_config $DS_CONFIG_PATH \
+    --zero-stage $ZERO_STAGE \
+    "
+
+CMD=" \
+    Megatron-DeepSpeed/pretrain_gpt.py \
+    --tensor-model-parallel-size $TP_SIZE \
+    --pipeline-model-parallel-size $PP_SIZE \
+    $GPT_ARGS \
+    $OUTPUT_ARGS \
+    --save $CHECKPOINT_PATH \
+    --load $CHECKPOINT_PATH \
+    --train-weighted-split-paths-path $TRAIN_DATA_PATH \
+    --valid-weighted-split-paths-path $VALID_DATA_PATH \
+    --data-impl mmap \
+     $DEEPSPEED_ARGS \
+    "
+
+echo $CMD
+
+echo "START $SLURM_JOBID: $(date)"
+
+# bash launch_srun_32.sh $CMD
+srun --label launch.sh $CMD
+
+echo "END $SLURM_JOBID: $(date)"

2b855b11bc4/tensorboard_2b855b11bval/events.out.tfevents.1682778100.nid005694.19446.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a1e7766fd7e5a76f990ff589d7f67ccdd9f850657fe217ee92d69250d174ef5c
+size 40

2b855b11bc4/tensorboard_2b855b11bval/events.out.tfevents.1682779220.nid005986.104630.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ffe36a8209a699e1366f8bf0983521ce5d3756f674f886ee9af78191d949c4ae
+size 40

2b855b11bc4/tensorboard_2b855b11bval/events.out.tfevents.1682786237.nid005066.77789.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b7b8cf5c4f2663d3a1108679cce0ada3e505c4c8f211a523a88e4c08ed73df2d
+size 40

2b855b11bc4/tensorboard_2b855b11bval/events.out.tfevents.1682787270.nid005066.86660.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f0a0c0f006d90afcddd58a7dd38694882d604c0f0906f986c99cfe1644244c10
+size 40

2b855b11bc4/tensorboard_2b855b11bval/events.out.tfevents.1682788108.nid005066.95179.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bb515ab134c8ca30e05be199f9018b3cf5ebcb867cafd2e918044dd331d39ede
+size 980

2b855b14bc4/latest

@@ -0,0 +1 @@
+global_step52452

2b855b14bc4/sbatch_2b855b14bval.sh

@@ -0,0 +1,168 @@
+#!/bin/bash
+#SBATCH --exclude=nid007571,nid007112,nid006774,nid007502,nid007506,nid007507,nid005145,nid006692,nid007218,nid007123,nid006124,nid006123,nid007496,nid007237,nid006852,nid007206,nid006947,nid007212,nid006977,nid007222,nid005444,nid007219,nid007493,nid007221,nid005300,nid005619,nid006118,nid005203,nid006113,nid006481,nid007077,nid005208,nid005207,nid005879,nid005901
+#SBATCH --nodes=32
+#SBATCH --ntasks-per-node=1
+#SBATCH --cpus-per-task=32
+#SBATCH --mem=256G
+#SBATCH -p standard-g
+#SBATCH -t 48:00:00
+#SBATCH --gpus-per-node=mi250:8
+#SBATCH --exclusive=user
+#SBATCH --hint=nomultithread
+#SBATCH --account=project_462000119
+#SBATCH -o logs/%j.out
+#SBATCH -e logs/%j.err
+
+VARIANT=2b855b14bval
+VARIANT_CKPT=lm1-2b8-55b-c4-repetitions/2b855b14bc4
+
+# if run without sbatch, invoke here
+if [ -z $SLURM_JOB_ID ]; then
+    mkdir -p logs
+    sbatch "$0"
+    exit
+fi
+
+set -euo pipefail
+
+# symlink logs/latest.out and logs/latest.err
+ln -f -s $SLURM_JOB_ID.out logs/latest.out
+ln -f -s $SLURM_JOB_ID.err logs/latest.err
+
+KILL_SWITCH_PATH=kill-switch-$VARIANT
+CHECKPOINT_PATH=$VARIANT_CKPT
+TENSORBOARD_PATH=tensorboard_$VARIANT
+
+# Data
+VOCAB_FILE="gpt2/vocab.json"
+MERGE_FILE="gpt2/merges.txt"
+#DATA_PATH="/scratch/project_462000119/data/pile/megatron_data/meg-gpt2_pile_text_document"
+TRAIN_DATA_PATH=train1b5.txt
+# "train: 1.0 0:1 /scratch/project_462000119/data/c4_subsampled/gpt2tok_c4_en_12B_text_document"
+VALID_DATA_PATH=val.txt
+# "validation: 1.0 0:1 /scratch/project_462000119/data/c4_validation/gpt2tok_c4validation_rerun_text_document"
+
+PP_SIZE=1
+TP_SIZE=1
+
+MICRO_BATCH_SIZE=2
+GRADIENT_ACCUMULATION_STEPS=1
+WORLD_SIZE=$((SLURM_GPUS_ON_NODE*SLURM_JOB_NUM_NODES))
+GLOBAL_BATCH_SIZE=$((MICRO_BATCH_SIZE*WORLD_SIZE*GRADIENT_ACCUMULATION_STEPS))
+
+# Model parameters
+source model_params.sh
+MODEL_PARAM=("${PARAM_2980M[@]}")
+NHIDDEN=${MODEL_PARAM[0]}
+FFN_HIDDEN_SIZE=${MODEL_PARAM[1]}
+KV_SIZE=${MODEL_PARAM[2]}
+NHEADS=${MODEL_PARAM[3]}
+NLAYERS=${MODEL_PARAM[4]}
+SEQ_LEN=2048
+
+echo "Model parameters: d_model $NHIDDEN ffw_size $FFN_HIDDEN_SIZE kv_size $KV_SIZE n_heads $NHEADS n_layers $NLAYERS"
+
+SAVE_INTERVAL=1000
+
+# Tokens: 35546190000
+# -> Samples: 17356538
+TRAIN_SAMPLES=1
+
+OPTIMIZER_ARGS=" \
+    --optimizer adam \
+    --adam-beta1 0.9 \
+    --adam-beta2 0.999 \
+    --adam-eps 1e-8 \
+    --lr 2e-4 \
+    --min-lr 2e-5 \
+    --lr-decay-style cosine \
+    --lr-decay-samples $TRAIN_SAMPLES \
+    --lr-warmup-samples 0 \
+    --clip-grad 1.0 \
+    --weight-decay 1e-1 \
+    --override-lr-scheduler \
+    --reset-progress \
+    --no-load-optim \
+    "
+
+GPT_ARGS=" \
+    --num-layers $NLAYERS \
+    --hidden-size $NHIDDEN \
+    --num-attention-heads $NHEADS \
+    --kv-channels $KV_SIZE \
+    --ffn-hidden-size $FFN_HIDDEN_SIZE \
+    --seq-length $SEQ_LEN \
+    --max-position-embeddings $SEQ_LEN \
+    --micro-batch-size $MICRO_BATCH_SIZE \
+    --global-batch-size $GLOBAL_BATCH_SIZE \
+    --train-samples $TRAIN_SAMPLES \
+    --vocab-file $VOCAB_FILE \
+    --merge-file $MERGE_FILE \
+    --clip-grad 1.0 \
+    --kill-switch-path $KILL_SWITCH_PATH \
+    --bf16 \
+    $OPTIMIZER_ARGS \
+    "
+
+OUTPUT_ARGS=" \
+    --log-interval 10 \
+    --save-interval $SAVE_INTERVAL \
+    --eval-interval 1 \
+    --eval-iters 100 \
+    --eval-only true \
+    --tensorboard-dir $TENSORBOARD_PATH \
+    --tensorboard-queue-size 5 \
+    --log-timers-to-tensorboard \
+    --log-batch-size-to-tensorboard \
+    --log-validation-ppl-to-tensorboard \
+    "
+
+ZERO_STAGE=0
+
+mkdir -p ds_configs
+DS_CONFIG_PATH="ds_configs/$SLURM_JOB_ID.json"
+
+cat <<EOF > $DS_CONFIG_PATH
+{
+    "train_micro_batch_size_per_gpu": $MICRO_BATCH_SIZE,
+    "train_batch_size": $GLOBAL_BATCH_SIZE,
+    "gradient_clipping": 1.0,
+    "zero_optimization": {
+        "stage": $ZERO_STAGE
+    },
+    "bf16": {
+        "enabled": true
+    },
+    "steps_per_print": 2000,
+    "wall_clock_breakdown": false
+}
+EOF
+
+DEEPSPEED_ARGS=" \
+    --deepspeed \
+    --deepspeed_config $DS_CONFIG_PATH \
+    --zero-stage $ZERO_STAGE \
+    "
+
+CMD=" \
+    Megatron-DeepSpeed/pretrain_gpt.py \
+    --tensor-model-parallel-size $TP_SIZE \
+    --pipeline-model-parallel-size $PP_SIZE \
+    $GPT_ARGS \
+    $OUTPUT_ARGS \
+    --save $CHECKPOINT_PATH \
+    --load $CHECKPOINT_PATH \
+    --train-weighted-split-paths-path $TRAIN_DATA_PATH \
+    --valid-weighted-split-paths-path $VALID_DATA_PATH \
+    --data-impl mmap \
+     $DEEPSPEED_ARGS \
+    "
+
+echo $CMD
+
+echo "START $SLURM_JOBID: $(date)"
+
+# bash launch_srun_32.sh $CMD
+srun --label launch.sh $CMD
+
+echo "END $SLURM_JOBID: $(date)"

2b855b14bc4/tensorboard_2b855b14bval/events.out.tfevents.1682778100.nid005609.106764.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ba684b3b91f424aa46d48e0bd01823a6628b06c00582474b8d0ba5e0265a7b58
+size 980

2b855b18bc4/latest

@@ -0,0 +1 @@
+global_step52452

2b855b18bc4/sbatch_2b855b18bval.sh

@@ -0,0 +1,168 @@
+#!/bin/bash
+#SBATCH --exclude=nid007571,nid007112,nid006774,nid007502,nid007506,nid007507,nid005145,nid006692,nid007218,nid007123,nid006124,nid006123,nid007496,nid007237,nid006852,nid007206,nid006947,nid007212,nid006977,nid007222,nid005444,nid007219,nid007493,nid007221,nid005300,nid005619,nid006118,nid005203,nid006113,nid006481,nid007077,nid005208,nid005207,nid005879,nid005901
+#SBATCH --nodes=32
+#SBATCH --ntasks-per-node=1
+#SBATCH --cpus-per-task=32
+#SBATCH --mem=256G
+#SBATCH -p standard-g
+#SBATCH -t 48:00:00
+#SBATCH --gpus-per-node=mi250:8
+#SBATCH --exclusive=user
+#SBATCH --hint=nomultithread
+#SBATCH --account=project_462000119
+#SBATCH -o logs/%j.out
+#SBATCH -e logs/%j.err
+
+VARIANT=2b855b18bval
+VARIANT_CKPT=lm1-2b8-55b-c4-repetitions/2b855b18bc4
+
+# if run without sbatch, invoke here
+if [ -z $SLURM_JOB_ID ]; then
+    mkdir -p logs
+    sbatch "$0"
+    exit
+fi
+
+set -euo pipefail
+
+# symlink logs/latest.out and logs/latest.err
+ln -f -s $SLURM_JOB_ID.out logs/latest.out
+ln -f -s $SLURM_JOB_ID.err logs/latest.err
+
+KILL_SWITCH_PATH=kill-switch-$VARIANT
+CHECKPOINT_PATH=$VARIANT_CKPT
+TENSORBOARD_PATH=tensorboard_$VARIANT
+
+# Data
+VOCAB_FILE="gpt2/vocab.json"
+MERGE_FILE="gpt2/merges.txt"
+#DATA_PATH="/scratch/project_462000119/data/pile/megatron_data/meg-gpt2_pile_text_document"
+TRAIN_DATA_PATH=train1b5.txt
+# "train: 1.0 0:1 /scratch/project_462000119/data/c4_subsampled/gpt2tok_c4_en_12B_text_document"
+VALID_DATA_PATH=val.txt
+# "validation: 1.0 0:1 /scratch/project_462000119/data/c4_validation/gpt2tok_c4validation_rerun_text_document"
+
+PP_SIZE=1
+TP_SIZE=1
+
+MICRO_BATCH_SIZE=2
+GRADIENT_ACCUMULATION_STEPS=1
+WORLD_SIZE=$((SLURM_GPUS_ON_NODE*SLURM_JOB_NUM_NODES))
+GLOBAL_BATCH_SIZE=$((MICRO_BATCH_SIZE*WORLD_SIZE*GRADIENT_ACCUMULATION_STEPS))
+
+# Model parameters
+source model_params.sh
+MODEL_PARAM=("${PARAM_2980M[@]}")
+NHIDDEN=${MODEL_PARAM[0]}
+FFN_HIDDEN_SIZE=${MODEL_PARAM[1]}
+KV_SIZE=${MODEL_PARAM[2]}
+NHEADS=${MODEL_PARAM[3]}
+NLAYERS=${MODEL_PARAM[4]}
+SEQ_LEN=2048
+
+echo "Model parameters: d_model $NHIDDEN ffw_size $FFN_HIDDEN_SIZE kv_size $KV_SIZE n_heads $NHEADS n_layers $NLAYERS"
+
+SAVE_INTERVAL=1000
+
+# Tokens: 35546190000
+# -> Samples: 17356538
+TRAIN_SAMPLES=1
+
+OPTIMIZER_ARGS=" \
+    --optimizer adam \
+    --adam-beta1 0.9 \
+    --adam-beta2 0.999 \
+    --adam-eps 1e-8 \
+    --lr 2e-4 \
+    --min-lr 2e-5 \
+    --lr-decay-style cosine \
+    --lr-decay-samples $TRAIN_SAMPLES \
+    --lr-warmup-samples 0 \
+    --clip-grad 1.0 \
+    --weight-decay 1e-1 \
+    --override-lr-scheduler \
+    --reset-progress \
+    --no-load-optim \
+    "
+
+GPT_ARGS=" \
+    --num-layers $NLAYERS \
+    --hidden-size $NHIDDEN \
+    --num-attention-heads $NHEADS \
+    --kv-channels $KV_SIZE \
+    --ffn-hidden-size $FFN_HIDDEN_SIZE \
+    --seq-length $SEQ_LEN \
+    --max-position-embeddings $SEQ_LEN \
+    --micro-batch-size $MICRO_BATCH_SIZE \
+    --global-batch-size $GLOBAL_BATCH_SIZE \
+    --train-samples $TRAIN_SAMPLES \
+    --vocab-file $VOCAB_FILE \
+    --merge-file $MERGE_FILE \
+    --clip-grad 1.0 \
+    --kill-switch-path $KILL_SWITCH_PATH \
+    --bf16 \
+    $OPTIMIZER_ARGS \
+    "
+
+OUTPUT_ARGS=" \
+    --log-interval 10 \
+    --save-interval $SAVE_INTERVAL \
+    --eval-interval 1 \
+    --eval-iters 100 \
+    --eval-only true \
+    --tensorboard-dir $TENSORBOARD_PATH \
+    --tensorboard-queue-size 5 \
+    --log-timers-to-tensorboard \
+    --log-batch-size-to-tensorboard \
+    --log-validation-ppl-to-tensorboard \
+    "
+
+ZERO_STAGE=0
+
+mkdir -p ds_configs
+DS_CONFIG_PATH="ds_configs/$SLURM_JOB_ID.json"
+
+cat <<EOF > $DS_CONFIG_PATH
+{
+    "train_micro_batch_size_per_gpu": $MICRO_BATCH_SIZE,
+    "train_batch_size": $GLOBAL_BATCH_SIZE,
+    "gradient_clipping": 1.0,
+    "zero_optimization": {
+        "stage": $ZERO_STAGE
+    },
+    "bf16": {
+        "enabled": true
+    },
+    "steps_per_print": 2000,
+    "wall_clock_breakdown": false
+}
+EOF
+
+DEEPSPEED_ARGS=" \
+    --deepspeed \
+    --deepspeed_config $DS_CONFIG_PATH \
+    --zero-stage $ZERO_STAGE \
+    "
+
+CMD=" \
+    Megatron-DeepSpeed/pretrain_gpt.py \
+    --tensor-model-parallel-size $TP_SIZE \
+    --pipeline-model-parallel-size $PP_SIZE \
+    $GPT_ARGS \
+    $OUTPUT_ARGS \
+    --save $CHECKPOINT_PATH \
+    --load $CHECKPOINT_PATH \
+    --train-weighted-split-paths-path $TRAIN_DATA_PATH \
+    --valid-weighted-split-paths-path $VALID_DATA_PATH \
+    --data-impl mmap \
+     $DEEPSPEED_ARGS \
+    "
+
+echo $CMD
+
+echo "START $SLURM_JOBID: $(date)"
+
+# bash launch_srun_32.sh $CMD
+srun --label launch.sh $CMD
+
+echo "END $SLURM_JOBID: $(date)"

2b855b18bc4/tensorboard_2b855b18bval/events.out.tfevents.1682778100.nid006583.11006.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:41f1df8567c43833877a3cfc2eaed6fce852ba5522954f1c79df8a6f3def82c8
+size 980

2b855b1b25c4/latest

@@ -0,0 +1 @@
+global_step52452

2b855b1b25c4/sbatch_2b855b1b25val.sh

@@ -0,0 +1,168 @@
+#!/bin/bash
+#SBATCH --exclude=nid007571,nid007112,nid006774,nid007502,nid007506,nid007507,nid005145,nid006692,nid007218,nid007123,nid006124,nid006123,nid007496,nid007237,nid006852,nid007206,nid006947,nid007212,nid006977,nid007222,nid005444,nid007219,nid007493,nid007221,nid005300,nid005619,nid006118,nid005203,nid006113,nid006481,nid007077,nid005208,nid005207,nid005879,nid005901
+#SBATCH --nodes=32
+#SBATCH --ntasks-per-node=1
+#SBATCH --cpus-per-task=32
+#SBATCH --mem=256G
+#SBATCH -p standard-g
+#SBATCH -t 48:00:00
+#SBATCH --gpus-per-node=mi250:8
+#SBATCH --exclusive=user
+#SBATCH --hint=nomultithread
+#SBATCH --account=project_462000119
+#SBATCH -o logs/%j.out
+#SBATCH -e logs/%j.err
+
+VARIANT=2b855b1b25val
+VARIANT_CKPT=lm1-2b8-55b-c4-repetitions/2b855b1b25c4
+
+# if run without sbatch, invoke here
+if [ -z $SLURM_JOB_ID ]; then
+    mkdir -p logs
+    sbatch "$0"
+    exit
+fi
+
+set -euo pipefail
+
+# symlink logs/latest.out and logs/latest.err
+ln -f -s $SLURM_JOB_ID.out logs/latest.out
+ln -f -s $SLURM_JOB_ID.err logs/latest.err
+
+KILL_SWITCH_PATH=kill-switch-$VARIANT
+CHECKPOINT_PATH=$VARIANT_CKPT
+TENSORBOARD_PATH=tensorboard_$VARIANT
+
+# Data
+VOCAB_FILE="gpt2/vocab.json"
+MERGE_FILE="gpt2/merges.txt"
+#DATA_PATH="/scratch/project_462000119/data/pile/megatron_data/meg-gpt2_pile_text_document"
+TRAIN_DATA_PATH=train1b5.txt
+# "train: 1.0 0:1 /scratch/project_462000119/data/c4_subsampled/gpt2tok_c4_en_12B_text_document"
+VALID_DATA_PATH=val.txt
+# "validation: 1.0 0:1 /scratch/project_462000119/data/c4_validation/gpt2tok_c4validation_rerun_text_document"
+
+PP_SIZE=1
+TP_SIZE=1
+
+MICRO_BATCH_SIZE=2
+GRADIENT_ACCUMULATION_STEPS=1
+WORLD_SIZE=$((SLURM_GPUS_ON_NODE*SLURM_JOB_NUM_NODES))
+GLOBAL_BATCH_SIZE=$((MICRO_BATCH_SIZE*WORLD_SIZE*GRADIENT_ACCUMULATION_STEPS))
+
+# Model parameters
+source model_params.sh
+MODEL_PARAM=("${PARAM_2980M[@]}")
+NHIDDEN=${MODEL_PARAM[0]}
+FFN_HIDDEN_SIZE=${MODEL_PARAM[1]}
+KV_SIZE=${MODEL_PARAM[2]}
+NHEADS=${MODEL_PARAM[3]}
+NLAYERS=${MODEL_PARAM[4]}
+SEQ_LEN=2048
+
+echo "Model parameters: d_model $NHIDDEN ffw_size $FFN_HIDDEN_SIZE kv_size $KV_SIZE n_heads $NHEADS n_layers $NLAYERS"
+
+SAVE_INTERVAL=1000
+
+# Tokens: 35546190000
+# -> Samples: 17356538
+TRAIN_SAMPLES=1
+
+OPTIMIZER_ARGS=" \
+    --optimizer adam \
+    --adam-beta1 0.9 \
+    --adam-beta2 0.999 \
+    --adam-eps 1e-8 \
+    --lr 2e-4 \
+    --min-lr 2e-5 \
+    --lr-decay-style cosine \
+    --lr-decay-samples $TRAIN_SAMPLES \
+    --lr-warmup-samples 0 \
+    --clip-grad 1.0 \
+    --weight-decay 1e-1 \
+    --override-lr-scheduler \
+    --reset-progress \
+    --no-load-optim \
+    "
+
+GPT_ARGS=" \
+    --num-layers $NLAYERS \
+    --hidden-size $NHIDDEN \
+    --num-attention-heads $NHEADS \
+    --kv-channels $KV_SIZE \
+    --ffn-hidden-size $FFN_HIDDEN_SIZE \
+    --seq-length $SEQ_LEN \
+    --max-position-embeddings $SEQ_LEN \
+    --micro-batch-size $MICRO_BATCH_SIZE \
+    --global-batch-size $GLOBAL_BATCH_SIZE \
+    --train-samples $TRAIN_SAMPLES \
+    --vocab-file $VOCAB_FILE \
+    --merge-file $MERGE_FILE \
+    --clip-grad 1.0 \
+    --kill-switch-path $KILL_SWITCH_PATH \
+    --bf16 \
+    $OPTIMIZER_ARGS \
+    "
+
+OUTPUT_ARGS=" \
+    --log-interval 10 \
+    --save-interval $SAVE_INTERVAL \
+    --eval-interval 1 \
+    --eval-iters 100 \
+    --eval-only true \
+    --tensorboard-dir $TENSORBOARD_PATH \
+    --tensorboard-queue-size 5 \
+    --log-timers-to-tensorboard \
+    --log-batch-size-to-tensorboard \
+    --log-validation-ppl-to-tensorboard \
+    "
+
+ZERO_STAGE=0
+
+mkdir -p ds_configs
+DS_CONFIG_PATH="ds_configs/$SLURM_JOB_ID.json"
+
+cat <<EOF > $DS_CONFIG_PATH
+{
+    "train_micro_batch_size_per_gpu": $MICRO_BATCH_SIZE,
+    "train_batch_size": $GLOBAL_BATCH_SIZE,
+    "gradient_clipping": 1.0,
+    "zero_optimization": {
+        "stage": $ZERO_STAGE
+    },
+    "bf16": {
+        "enabled": true
+    },
+    "steps_per_print": 2000,
+    "wall_clock_breakdown": false
+}
+EOF
+
+DEEPSPEED_ARGS=" \
+    --deepspeed \
+    --deepspeed_config $DS_CONFIG_PATH \
+    --zero-stage $ZERO_STAGE \
+    "
+
+CMD=" \
+    Megatron-DeepSpeed/pretrain_gpt.py \
+    --tensor-model-parallel-size $TP_SIZE \
+    --pipeline-model-parallel-size $PP_SIZE \
+    $GPT_ARGS \
+    $OUTPUT_ARGS \
+    --save $CHECKPOINT_PATH \
+    --load $CHECKPOINT_PATH \
+    --train-weighted-split-paths-path $TRAIN_DATA_PATH \
+    --valid-weighted-split-paths-path $VALID_DATA_PATH \
+    --data-impl mmap \
+     $DEEPSPEED_ARGS \
+    "
+
+echo $CMD
+
+echo "START $SLURM_JOBID: $(date)"
+
+# bash launch_srun_32.sh $CMD
+srun --label launch.sh $CMD
+
+echo "END $SLURM_JOBID: $(date)"

2b855b1b25c4/tensorboard_2b855b1b25val/events.out.tfevents.1682778212.nid006180.39887.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f7845567542e026d5a4005e05c303fe50afc829b64692ecfda4a4f722292616f
+size 980

2b855b28bc4/latest

@@ -0,0 +1 @@
+global_step52452

2b855b28bc4/sbatch_2b855b28bval.sh

@@ -0,0 +1,168 @@
+#!/bin/bash
+#SBATCH --exclude=nid007571,nid007112,nid006774,nid007502,nid007506,nid007507,nid005145,nid006692,nid007218,nid007123,nid006124,nid006123,nid007496,nid007237,nid006852,nid007206,nid006947,nid007212,nid006977,nid007222,nid005444,nid007219,nid007493,nid007221,nid005300,nid005619,nid006118,nid005203,nid006113,nid006481,nid007077,nid005208,nid005207,nid005879,nid005901
+#SBATCH --nodes=32
+#SBATCH --ntasks-per-node=1
+#SBATCH --cpus-per-task=32
+#SBATCH --mem=256G
+#SBATCH -p standard-g
+#SBATCH -t 48:00:00
+#SBATCH --gpus-per-node=mi250:8
+#SBATCH --exclusive=user
+#SBATCH --hint=nomultithread
+#SBATCH --account=project_462000119
+#SBATCH -o logs/%j.out
+#SBATCH -e logs/%j.err
+
+VARIANT=2b855b28bval
+VARIANT_CKPT=lm1-2b8-55b-c4-repetitions/2b855b28bc4
+
+# if run without sbatch, invoke here
+if [ -z $SLURM_JOB_ID ]; then
+    mkdir -p logs
+    sbatch "$0"
+    exit
+fi
+
+set -euo pipefail
+
+# symlink logs/latest.out and logs/latest.err
+ln -f -s $SLURM_JOB_ID.out logs/latest.out
+ln -f -s $SLURM_JOB_ID.err logs/latest.err
+
+KILL_SWITCH_PATH=kill-switch-$VARIANT
+CHECKPOINT_PATH=$VARIANT_CKPT
+TENSORBOARD_PATH=tensorboard_$VARIANT
+
+# Data
+VOCAB_FILE="gpt2/vocab.json"
+MERGE_FILE="gpt2/merges.txt"
+#DATA_PATH="/scratch/project_462000119/data/pile/megatron_data/meg-gpt2_pile_text_document"
+TRAIN_DATA_PATH=train1b5.txt
+# "train: 1.0 0:1 /scratch/project_462000119/data/c4_subsampled/gpt2tok_c4_en_12B_text_document"
+VALID_DATA_PATH=val.txt
+# "validation: 1.0 0:1 /scratch/project_462000119/data/c4_validation/gpt2tok_c4validation_rerun_text_document"
+
+PP_SIZE=1
+TP_SIZE=1
+
+MICRO_BATCH_SIZE=2
+GRADIENT_ACCUMULATION_STEPS=1
+WORLD_SIZE=$((SLURM_GPUS_ON_NODE*SLURM_JOB_NUM_NODES))
+GLOBAL_BATCH_SIZE=$((MICRO_BATCH_SIZE*WORLD_SIZE*GRADIENT_ACCUMULATION_STEPS))
+
+# Model parameters
+source model_params.sh
+MODEL_PARAM=("${PARAM_2980M[@]}")
+NHIDDEN=${MODEL_PARAM[0]}
+FFN_HIDDEN_SIZE=${MODEL_PARAM[1]}
+KV_SIZE=${MODEL_PARAM[2]}
+NHEADS=${MODEL_PARAM[3]}
+NLAYERS=${MODEL_PARAM[4]}
+SEQ_LEN=2048
+
+echo "Model parameters: d_model $NHIDDEN ffw_size $FFN_HIDDEN_SIZE kv_size $KV_SIZE n_heads $NHEADS n_layers $NLAYERS"
+
+SAVE_INTERVAL=1000
+
+# Tokens: 35546190000
+# -> Samples: 17356538
+TRAIN_SAMPLES=1
+
+OPTIMIZER_ARGS=" \
+    --optimizer adam \
+    --adam-beta1 0.9 \
+    --adam-beta2 0.999 \
+    --adam-eps 1e-8 \
+    --lr 2e-4 \
+    --min-lr 2e-5 \
+    --lr-decay-style cosine \
+    --lr-decay-samples $TRAIN_SAMPLES \
+    --lr-warmup-samples 0 \
+    --clip-grad 1.0 \
+    --weight-decay 1e-1 \
+    --override-lr-scheduler \
+    --reset-progress \
+    --no-load-optim \
+    "
+
+GPT_ARGS=" \
+    --num-layers $NLAYERS \
+    --hidden-size $NHIDDEN \
+    --num-attention-heads $NHEADS \
+    --kv-channels $KV_SIZE \
+    --ffn-hidden-size $FFN_HIDDEN_SIZE \
+    --seq-length $SEQ_LEN \
+    --max-position-embeddings $SEQ_LEN \
+    --micro-batch-size $MICRO_BATCH_SIZE \
+    --global-batch-size $GLOBAL_BATCH_SIZE \
+    --train-samples $TRAIN_SAMPLES \
+    --vocab-file $VOCAB_FILE \
+    --merge-file $MERGE_FILE \
+    --clip-grad 1.0 \
+    --kill-switch-path $KILL_SWITCH_PATH \
+    --bf16 \
+    $OPTIMIZER_ARGS \
+    "
+
+OUTPUT_ARGS=" \
+    --log-interval 10 \
+    --save-interval $SAVE_INTERVAL \
+    --eval-interval 1 \
+    --eval-iters 100 \
+    --eval-only true \
+    --tensorboard-dir $TENSORBOARD_PATH \
+    --tensorboard-queue-size 5 \
+    --log-timers-to-tensorboard \
+    --log-batch-size-to-tensorboard \
+    --log-validation-ppl-to-tensorboard \
+    "
+
+ZERO_STAGE=0
+
+mkdir -p ds_configs
+DS_CONFIG_PATH="ds_configs/$SLURM_JOB_ID.json"
+
+cat <<EOF > $DS_CONFIG_PATH
+{
+    "train_micro_batch_size_per_gpu": $MICRO_BATCH_SIZE,
+    "train_batch_size": $GLOBAL_BATCH_SIZE,
+    "gradient_clipping": 1.0,
+    "zero_optimization": {
+        "stage": $ZERO_STAGE
+    },
+    "bf16": {
+        "enabled": true
+    },
+    "steps_per_print": 2000,
+    "wall_clock_breakdown": false
+}
+EOF
+
+DEEPSPEED_ARGS=" \
+    --deepspeed \
+    --deepspeed_config $DS_CONFIG_PATH \
+    --zero-stage $ZERO_STAGE \
+    "
+
+CMD=" \
+    Megatron-DeepSpeed/pretrain_gpt.py \
+    --tensor-model-parallel-size $TP_SIZE \
+    --pipeline-model-parallel-size $PP_SIZE \
+    $GPT_ARGS \
+    $OUTPUT_ARGS \
+    --save $CHECKPOINT_PATH \
+    --load $CHECKPOINT_PATH \
+    --train-weighted-split-paths-path $TRAIN_DATA_PATH \
+    --valid-weighted-split-paths-path $VALID_DATA_PATH \
+    --data-impl mmap \
+     $DEEPSPEED_ARGS \
+    "
+
+echo $CMD
+
+echo "START $SLURM_JOBID: $(date)"
+
+# bash launch_srun_32.sh $CMD
+srun --label launch.sh $CMD
+
+echo "END $SLURM_JOBID: $(date)"

2b855b28bc4/tensorboard_2b855b28bval/events.out.tfevents.1682778100.nid005986.93930.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4baa4df65d36d2a7027a669f6b33fb5048247296ce39bde51f45843f29f2416e
+size 980

2b855b4bc4/latest

@@ -0,0 +1 @@
+global_step52452

2b855b4bc4/sbatch_2b855b4bval.sh

@@ -0,0 +1,168 @@
+#!/bin/bash
+#SBATCH --exclude=nid007571,nid007112,nid006774,nid007502,nid007506,nid007507,nid005145,nid006692,nid007218,nid007123,nid006124,nid006123,nid007496,nid007237,nid006852,nid007206,nid006947,nid007212,nid006977,nid007222,nid005444,nid007219,nid007493,nid007221,nid005300,nid005619,nid006118,nid005203,nid006113,nid006481,nid007077,nid005208,nid005207,nid005879,nid005901
+#SBATCH --nodes=32
+#SBATCH --ntasks-per-node=1
+#SBATCH --cpus-per-task=32
+#SBATCH --mem=256G
+#SBATCH -p standard-g
+#SBATCH -t 48:00:00
+#SBATCH --gpus-per-node=mi250:8
+#SBATCH --exclusive=user
+#SBATCH --hint=nomultithread
+#SBATCH --account=project_462000119
+#SBATCH -o logs/%j.out
+#SBATCH -e logs/%j.err
+
+VARIANT=2b855b4bval
+VARIANT_CKPT=lm1-2b8-55b-c4-repetitions/2b855b4bc4
+
+# if run without sbatch, invoke here
+if [ -z $SLURM_JOB_ID ]; then
+    mkdir -p logs
+    sbatch "$0"
+    exit
+fi
+
+set -euo pipefail
+
+# symlink logs/latest.out and logs/latest.err
+ln -f -s $SLURM_JOB_ID.out logs/latest.out
+ln -f -s $SLURM_JOB_ID.err logs/latest.err
+
+KILL_SWITCH_PATH=kill-switch-$VARIANT
+CHECKPOINT_PATH=$VARIANT_CKPT
+TENSORBOARD_PATH=tensorboard_$VARIANT
+
+# Data
+VOCAB_FILE="gpt2/vocab.json"
+MERGE_FILE="gpt2/merges.txt"
+#DATA_PATH="/scratch/project_462000119/data/pile/megatron_data/meg-gpt2_pile_text_document"
+TRAIN_DATA_PATH=train1b5.txt
+# "train: 1.0 0:1 /scratch/project_462000119/data/c4_subsampled/gpt2tok_c4_en_12B_text_document"
+VALID_DATA_PATH=val.txt
+# "validation: 1.0 0:1 /scratch/project_462000119/data/c4_validation/gpt2tok_c4validation_rerun_text_document"
+
+PP_SIZE=1
+TP_SIZE=1
+
+MICRO_BATCH_SIZE=2
+GRADIENT_ACCUMULATION_STEPS=1
+WORLD_SIZE=$((SLURM_GPUS_ON_NODE*SLURM_JOB_NUM_NODES))
+GLOBAL_BATCH_SIZE=$((MICRO_BATCH_SIZE*WORLD_SIZE*GRADIENT_ACCUMULATION_STEPS))
+
+# Model parameters
+source model_params.sh
+MODEL_PARAM=("${PARAM_2980M[@]}")
+NHIDDEN=${MODEL_PARAM[0]}
+FFN_HIDDEN_SIZE=${MODEL_PARAM[1]}
+KV_SIZE=${MODEL_PARAM[2]}
+NHEADS=${MODEL_PARAM[3]}
+NLAYERS=${MODEL_PARAM[4]}
+SEQ_LEN=2048
+
+echo "Model parameters: d_model $NHIDDEN ffw_size $FFN_HIDDEN_SIZE kv_size $KV_SIZE n_heads $NHEADS n_layers $NLAYERS"
+
+SAVE_INTERVAL=1000
+
+# Tokens: 35546190000
+# -> Samples: 17356538
+TRAIN_SAMPLES=1
+
+OPTIMIZER_ARGS=" \
+    --optimizer adam \
+    --adam-beta1 0.9 \
+    --adam-beta2 0.999 \
+    --adam-eps 1e-8 \
+    --lr 2e-4 \
+    --min-lr 2e-5 \
+    --lr-decay-style cosine \
+    --lr-decay-samples $TRAIN_SAMPLES \
+    --lr-warmup-samples 0 \
+    --clip-grad 1.0 \
+    --weight-decay 1e-1 \
+    --override-lr-scheduler \
+    --reset-progress \
+    --no-load-optim \
+    "
+
+GPT_ARGS=" \
+    --num-layers $NLAYERS \
+    --hidden-size $NHIDDEN \
+    --num-attention-heads $NHEADS \
+    --kv-channels $KV_SIZE \
+    --ffn-hidden-size $FFN_HIDDEN_SIZE \
+    --seq-length $SEQ_LEN \
+    --max-position-embeddings $SEQ_LEN \
+    --micro-batch-size $MICRO_BATCH_SIZE \
+    --global-batch-size $GLOBAL_BATCH_SIZE \
+    --train-samples $TRAIN_SAMPLES \
+    --vocab-file $VOCAB_FILE \
+    --merge-file $MERGE_FILE \
+    --clip-grad 1.0 \
+    --kill-switch-path $KILL_SWITCH_PATH \
+    --bf16 \
+    $OPTIMIZER_ARGS \
+    "
+
+OUTPUT_ARGS=" \
+    --log-interval 10 \
+    --save-interval $SAVE_INTERVAL \
+    --eval-interval 1 \
+    --eval-iters 100 \
+    --eval-only true \
+    --tensorboard-dir $TENSORBOARD_PATH \
+    --tensorboard-queue-size 5 \
+    --log-timers-to-tensorboard \
+    --log-batch-size-to-tensorboard \
+    --log-validation-ppl-to-tensorboard \
+    "
+
+ZERO_STAGE=0
+
+mkdir -p ds_configs
+DS_CONFIG_PATH="ds_configs/$SLURM_JOB_ID.json"
+
+cat <<EOF > $DS_CONFIG_PATH
+{
+    "train_micro_batch_size_per_gpu": $MICRO_BATCH_SIZE,
+    "train_batch_size": $GLOBAL_BATCH_SIZE,
+    "gradient_clipping": 1.0,
+    "zero_optimization": {
+        "stage": $ZERO_STAGE
+    },
+    "bf16": {
+        "enabled": true
+    },
+    "steps_per_print": 2000,
+    "wall_clock_breakdown": false
+}
+EOF
+
+DEEPSPEED_ARGS=" \
+    --deepspeed \
+    --deepspeed_config $DS_CONFIG_PATH \
+    --zero-stage $ZERO_STAGE \
+    "
+
+CMD=" \
+    Megatron-DeepSpeed/pretrain_gpt.py \
+    --tensor-model-parallel-size $TP_SIZE \
+    --pipeline-model-parallel-size $PP_SIZE \
+    $GPT_ARGS \
+    $OUTPUT_ARGS \
+    --save $CHECKPOINT_PATH \
+    --load $CHECKPOINT_PATH \
+    --train-weighted-split-paths-path $TRAIN_DATA_PATH \
+    --valid-weighted-split-paths-path $VALID_DATA_PATH \
+    --data-impl mmap \
+     $DEEPSPEED_ARGS \
+    "
+
+echo $CMD
+
+echo "START $SLURM_JOBID: $(date)"
+
+# bash launch_srun_32.sh $CMD
+srun --label launch.sh $CMD
+
+echo "END $SLURM_JOBID: $(date)"

2b855b4bc4/tensorboard_2b855b4bval/events.out.tfevents.1682778280.nid006557.82298.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4b83a0b5b03003664445af2aa49ab29196c2dd2c6b534f1179dcf7c5a9c8455c
+size 980

2b855b55bc4/latest

@@ -0,0 +1 @@
+global_step52452

2b855b55bc4/sbatch_2b855b55bval.sh

@@ -0,0 +1,168 @@
+#!/bin/bash
+#SBATCH --exclude=nid007571,nid007112,nid006774,nid007502,nid007506,nid007507,nid005145,nid006692,nid007218,nid007123,nid006124,nid006123,nid007496,nid007237,nid006852,nid007206,nid006947,nid007212,nid006977,nid007222,nid005444,nid007219,nid007493,nid007221,nid005300,nid005619,nid006118,nid005203,nid006113,nid006481,nid007077,nid005208,nid005207,nid005879,nid005901
+#SBATCH --nodes=32
+#SBATCH --ntasks-per-node=1
+#SBATCH --cpus-per-task=32
+#SBATCH --mem=256G
+#SBATCH -p standard-g
+#SBATCH -t 48:00:00
+#SBATCH --gpus-per-node=mi250:8
+#SBATCH --exclusive=user
+#SBATCH --hint=nomultithread
+#SBATCH --account=project_462000119
+#SBATCH -o logs/%j.out
+#SBATCH -e logs/%j.err
+
+VARIANT=2b855b55bval
+VARIANT_CKPT=lm1-2b8-55b-c4-repetitions/2b855b55bc4
+
+# if run without sbatch, invoke here
+if [ -z $SLURM_JOB_ID ]; then
+    mkdir -p logs
+    sbatch "$0"
+    exit
+fi
+
+set -euo pipefail
+
+# symlink logs/latest.out and logs/latest.err
+ln -f -s $SLURM_JOB_ID.out logs/latest.out
+ln -f -s $SLURM_JOB_ID.err logs/latest.err
+
+KILL_SWITCH_PATH=kill-switch-$VARIANT
+CHECKPOINT_PATH=$VARIANT_CKPT
+TENSORBOARD_PATH=tensorboard_$VARIANT
+
+# Data
+VOCAB_FILE="gpt2/vocab.json"
+MERGE_FILE="gpt2/merges.txt"
+#DATA_PATH="/scratch/project_462000119/data/pile/megatron_data/meg-gpt2_pile_text_document"
+TRAIN_DATA_PATH=train1b5.txt
+# "train: 1.0 0:1 /scratch/project_462000119/data/c4_subsampled/gpt2tok_c4_en_12B_text_document"
+VALID_DATA_PATH=val.txt
+# "validation: 1.0 0:1 /scratch/project_462000119/data/c4_validation/gpt2tok_c4validation_rerun_text_document"
+
+PP_SIZE=1
+TP_SIZE=1
+
+MICRO_BATCH_SIZE=2
+GRADIENT_ACCUMULATION_STEPS=1
+WORLD_SIZE=$((SLURM_GPUS_ON_NODE*SLURM_JOB_NUM_NODES))
+GLOBAL_BATCH_SIZE=$((MICRO_BATCH_SIZE*WORLD_SIZE*GRADIENT_ACCUMULATION_STEPS))
+
+# Model parameters
+source model_params.sh
+MODEL_PARAM=("${PARAM_2980M[@]}")
+NHIDDEN=${MODEL_PARAM[0]}
+FFN_HIDDEN_SIZE=${MODEL_PARAM[1]}
+KV_SIZE=${MODEL_PARAM[2]}
+NHEADS=${MODEL_PARAM[3]}
+NLAYERS=${MODEL_PARAM[4]}
+SEQ_LEN=2048
+
+echo "Model parameters: d_model $NHIDDEN ffw_size $FFN_HIDDEN_SIZE kv_size $KV_SIZE n_heads $NHEADS n_layers $NLAYERS"
+
+SAVE_INTERVAL=1000
+
+# Tokens: 35546190000
+# -> Samples: 17356538
+TRAIN_SAMPLES=1
+
+OPTIMIZER_ARGS=" \
+    --optimizer adam \
+    --adam-beta1 0.9 \
+    --adam-beta2 0.999 \
+    --adam-eps 1e-8 \
+    --lr 2e-4 \
+    --min-lr 2e-5 \
+    --lr-decay-style cosine \
+    --lr-decay-samples $TRAIN_SAMPLES \
+    --lr-warmup-samples 0 \
+    --clip-grad 1.0 \
+    --weight-decay 1e-1 \
+    --override-lr-scheduler \
+    --reset-progress \
+    --no-load-optim \
+    "
+
+GPT_ARGS=" \
+    --num-layers $NLAYERS \
+    --hidden-size $NHIDDEN \
+    --num-attention-heads $NHEADS \
+    --kv-channels $KV_SIZE \
+    --ffn-hidden-size $FFN_HIDDEN_SIZE \
+    --seq-length $SEQ_LEN \
+    --max-position-embeddings $SEQ_LEN \
+    --micro-batch-size $MICRO_BATCH_SIZE \
+    --global-batch-size $GLOBAL_BATCH_SIZE \
+    --train-samples $TRAIN_SAMPLES \
+    --vocab-file $VOCAB_FILE \
+    --merge-file $MERGE_FILE \
+    --clip-grad 1.0 \
+    --kill-switch-path $KILL_SWITCH_PATH \
+    --bf16 \
+    $OPTIMIZER_ARGS \
+    "
+
+OUTPUT_ARGS=" \
+    --log-interval 10 \
+    --save-interval $SAVE_INTERVAL \
+    --eval-interval 1 \
+    --eval-iters 100 \
+    --eval-only true \
+    --tensorboard-dir $TENSORBOARD_PATH \
+    --tensorboard-queue-size 5 \
+    --log-timers-to-tensorboard \
+    --log-batch-size-to-tensorboard \
+    --log-validation-ppl-to-tensorboard \
+    "
+
+ZERO_STAGE=0
+
+mkdir -p ds_configs
+DS_CONFIG_PATH="ds_configs/$SLURM_JOB_ID.json"
+
+cat <<EOF > $DS_CONFIG_PATH
+{
+    "train_micro_batch_size_per_gpu": $MICRO_BATCH_SIZE,
+    "train_batch_size": $GLOBAL_BATCH_SIZE,
+    "gradient_clipping": 1.0,
+    "zero_optimization": {
+        "stage": $ZERO_STAGE
+    },
+    "bf16": {
+        "enabled": true
+    },
+    "steps_per_print": 2000,
+    "wall_clock_breakdown": false
+}
+EOF
+
+DEEPSPEED_ARGS=" \
+    --deepspeed \
+    --deepspeed_config $DS_CONFIG_PATH \
+    --zero-stage $ZERO_STAGE \
+    "
+
+CMD=" \
+    Megatron-DeepSpeed/pretrain_gpt.py \
+    --tensor-model-parallel-size $TP_SIZE \
+    --pipeline-model-parallel-size $PP_SIZE \
+    $GPT_ARGS \
+    $OUTPUT_ARGS \
+    --save $CHECKPOINT_PATH \
+    --load $CHECKPOINT_PATH \
+    --train-weighted-split-paths-path $TRAIN_DATA_PATH \
+    --valid-weighted-split-paths-path $VALID_DATA_PATH \
+    --data-impl mmap \
+     $DEEPSPEED_ARGS \
+    "
+
+echo $CMD
+
+echo "START $SLURM_JOBID: $(date)"
+
+# bash launch_srun_32.sh $CMD
+srun --label launch.sh $CMD
+
+echo "END $SLURM_JOBID: $(date)"

2b855b55bc4/tensorboard_2b855b55bval/events.out.tfevents.1682762738.nid007070.128063.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3b7e10361e13452c7cb69449bac0308f1fc721d7453cc3263e4e02cf2f9ea9b9
+size 40

2b855b55bc4/tensorboard_2b855b55bval/events.out.tfevents.1682776426.nid005986.78920.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:44090d2e1f29367a90741ead21f5cc1e2297464f1c6bdddafa0c245b2434f6d3
+size 980

2b855b9bc4/latest

@@ -0,0 +1 @@
+global_step52452

2b855b9bc4/sbatch_2b855b9bval.sh

@@ -0,0 +1,168 @@
+#!/bin/bash
+#SBATCH --exclude=nid007571,nid007112,nid006774,nid007502,nid007506,nid007507,nid005145,nid006692,nid007218,nid007123,nid006124,nid006123,nid007496,nid007237,nid006852,nid007206,nid006947,nid007212,nid006977,nid007222,nid005444,nid007219,nid007493,nid007221,nid005300,nid005619,nid006118,nid005203,nid006113,nid006481,nid007077,nid005208,nid005207,nid005879,nid005901
+#SBATCH --nodes=32
+#SBATCH --ntasks-per-node=1
+#SBATCH --cpus-per-task=32
+#SBATCH --mem=256G
+#SBATCH -p standard-g
+#SBATCH -t 48:00:00
+#SBATCH --gpus-per-node=mi250:8
+#SBATCH --exclusive=user
+#SBATCH --hint=nomultithread
+#SBATCH --account=project_462000119
+#SBATCH -o logs/%j.out
+#SBATCH -e logs/%j.err
+
+VARIANT=2b855b9bval
+VARIANT_CKPT=lm1-2b8-55b-c4-repetitions/2b855b9bc4
+
+# if run without sbatch, invoke here
+if [ -z $SLURM_JOB_ID ]; then
+    mkdir -p logs
+    sbatch "$0"
+    exit
+fi
+
+set -euo pipefail
+
+# symlink logs/latest.out and logs/latest.err
+ln -f -s $SLURM_JOB_ID.out logs/latest.out
+ln -f -s $SLURM_JOB_ID.err logs/latest.err
+
+KILL_SWITCH_PATH=kill-switch-$VARIANT
+CHECKPOINT_PATH=$VARIANT_CKPT
+TENSORBOARD_PATH=tensorboard_$VARIANT
+
+# Data
+VOCAB_FILE="gpt2/vocab.json"
+MERGE_FILE="gpt2/merges.txt"
+#DATA_PATH="/scratch/project_462000119/data/pile/megatron_data/meg-gpt2_pile_text_document"
+TRAIN_DATA_PATH=train1b5.txt
+# "train: 1.0 0:1 /scratch/project_462000119/data/c4_subsampled/gpt2tok_c4_en_12B_text_document"
+VALID_DATA_PATH=val.txt
+# "validation: 1.0 0:1 /scratch/project_462000119/data/c4_validation/gpt2tok_c4validation_rerun_text_document"
+
+PP_SIZE=1
+TP_SIZE=1
+
+MICRO_BATCH_SIZE=2
+GRADIENT_ACCUMULATION_STEPS=1
+WORLD_SIZE=$((SLURM_GPUS_ON_NODE*SLURM_JOB_NUM_NODES))
+GLOBAL_BATCH_SIZE=$((MICRO_BATCH_SIZE*WORLD_SIZE*GRADIENT_ACCUMULATION_STEPS))
+
+# Model parameters
+source model_params.sh
+MODEL_PARAM=("${PARAM_2980M[@]}")
+NHIDDEN=${MODEL_PARAM[0]}
+FFN_HIDDEN_SIZE=${MODEL_PARAM[1]}
+KV_SIZE=${MODEL_PARAM[2]}
+NHEADS=${MODEL_PARAM[3]}
+NLAYERS=${MODEL_PARAM[4]}
+SEQ_LEN=2048
+
+echo "Model parameters: d_model $NHIDDEN ffw_size $FFN_HIDDEN_SIZE kv_size $KV_SIZE n_heads $NHEADS n_layers $NLAYERS"
+
+SAVE_INTERVAL=1000
+
+# Tokens: 35546190000
+# -> Samples: 17356538
+TRAIN_SAMPLES=1
+
+OPTIMIZER_ARGS=" \
+    --optimizer adam \
+    --adam-beta1 0.9 \
+    --adam-beta2 0.999 \
+    --adam-eps 1e-8 \
+    --lr 2e-4 \
+    --min-lr 2e-5 \
+    --lr-decay-style cosine \
+    --lr-decay-samples $TRAIN_SAMPLES \
+    --lr-warmup-samples 0 \
+    --clip-grad 1.0 \
+    --weight-decay 1e-1 \
+    --override-lr-scheduler \
+    --reset-progress \
+    --no-load-optim \
+    "
+
+GPT_ARGS=" \
+    --num-layers $NLAYERS \
+    --hidden-size $NHIDDEN \
+    --num-attention-heads $NHEADS \
+    --kv-channels $KV_SIZE \
+    --ffn-hidden-size $FFN_HIDDEN_SIZE \
+    --seq-length $SEQ_LEN \
+    --max-position-embeddings $SEQ_LEN \
+    --micro-batch-size $MICRO_BATCH_SIZE \
+    --global-batch-size $GLOBAL_BATCH_SIZE \
+    --train-samples $TRAIN_SAMPLES \
+    --vocab-file $VOCAB_FILE \
+    --merge-file $MERGE_FILE \
+    --clip-grad 1.0 \
+    --kill-switch-path $KILL_SWITCH_PATH \
+    --bf16 \
+    $OPTIMIZER_ARGS \
+    "
+
+OUTPUT_ARGS=" \
+    --log-interval 10 \
+    --save-interval $SAVE_INTERVAL \
+    --eval-interval 1 \
+    --eval-iters 100 \
+    --eval-only true \
+    --tensorboard-dir $TENSORBOARD_PATH \
+    --tensorboard-queue-size 5 \
+    --log-timers-to-tensorboard \
+    --log-batch-size-to-tensorboard \
+    --log-validation-ppl-to-tensorboard \
+    "
+
+ZERO_STAGE=0
+
+mkdir -p ds_configs
+DS_CONFIG_PATH="ds_configs/$SLURM_JOB_ID.json"
+
+cat <<EOF > $DS_CONFIG_PATH
+{
+    "train_micro_batch_size_per_gpu": $MICRO_BATCH_SIZE,
+    "train_batch_size": $GLOBAL_BATCH_SIZE,
+    "gradient_clipping": 1.0,
+    "zero_optimization": {
+        "stage": $ZERO_STAGE
+    },
+    "bf16": {
+        "enabled": true
+    },
+    "steps_per_print": 2000,
+    "wall_clock_breakdown": false
+}
+EOF
+
+DEEPSPEED_ARGS=" \
+    --deepspeed \
+    --deepspeed_config $DS_CONFIG_PATH \
+    --zero-stage $ZERO_STAGE \
+    "
+
+CMD=" \
+    Megatron-DeepSpeed/pretrain_gpt.py \
+    --tensor-model-parallel-size $TP_SIZE \
+    --pipeline-model-parallel-size $PP_SIZE \
+    $GPT_ARGS \
+    $OUTPUT_ARGS \
+    --save $CHECKPOINT_PATH \
+    --load $CHECKPOINT_PATH \
+    --train-weighted-split-paths-path $TRAIN_DATA_PATH \
+    --valid-weighted-split-paths-path $VALID_DATA_PATH \
+    --data-impl mmap \
+     $DEEPSPEED_ARGS \
+    "
+
+echo $CMD
+
+echo "START $SLURM_JOBID: $(date)"
+
+# bash launch_srun_32.sh $CMD
+srun --label launch.sh $CMD
+
+echo "END $SLURM_JOBID: $(date)"

2b855b9bc4/tensorboard_2b855b9bval/events.out.tfevents.1682778178.nid005714.110139.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:31780306faf7afbd4416161d58f3cdd748c644b474000eb3b50e2d8a6f6e8c15
+size 980