Muennighoff commited on
Commit
112233b
1 Parent(s): 046a6f8
This view is limited to 50 files because it contains too many changes.   See raw diff
Files changed (50) hide show
  1. sbatch_4b284b12bc4seed1.sh +164 -0
  2. sbatch_4b284b12bc4seed2.sh +164 -0
  3. sbatch_4b284b12bc4seed3.sh +164 -0
  4. sbatch_4b284b12bc4seed4.sh +164 -0
  5. sbatch_4b284b17bc4seed1.sh +164 -0
  6. sbatch_4b284b17bc4seed2.sh +164 -0
  7. sbatch_4b284b17bc4seed3.sh +164 -0
  8. sbatch_4b284b17bc4seed4.sh +164 -0
  9. sbatch_4b284b21bc4seed1.sh +164 -0
  10. sbatch_4b284b21bc4seed2.sh +164 -0
  11. sbatch_4b284b21bc4seed3.sh +164 -0
  12. sbatch_4b284b21bc4seed4.sh +164 -0
  13. sbatch_4b284b28bc4seed1.sh +164 -0
  14. sbatch_4b284b28bc4seed2.sh +164 -0
  15. sbatch_4b284b28bc4seed3.sh +164 -0
  16. sbatch_4b284b28bc4seed4.sh +164 -0
  17. sbatch_4b284b42bc4seed1.sh +164 -0
  18. sbatch_4b284b42bc4seed2.sh +164 -0
  19. sbatch_4b284b42bc4seed3.sh +164 -0
  20. sbatch_4b284b42bc4seed4.sh +164 -0
  21. sbatch_4b284b84bc4v2seed1.sh +164 -0
  22. sbatch_4b284b84bc4v2seed2.sh +164 -0
  23. sbatch_4b284b84bc4v2seed3.sh +164 -0
  24. sbatch_4b284b84bc4v2seed4.sh +164 -0
  25. tensorboard/tensorboard_4b284b12bc4seed1/events.out.tfevents.1676595687.nid006969.74604.0 +3 -0
  26. tensorboard/tensorboard_4b284b12bc4seed1/events.out.tfevents.1676596170.nid006963.99099.0 +3 -0
  27. tensorboard/tensorboard_4b284b12bc4seed1/events.out.tfevents.1676596625.nid006963.104051.0 +3 -0
  28. tensorboard/tensorboard_4b284b12bc4seed1/events.out.tfevents.1676597074.nid006963.112349.0 +3 -0
  29. tensorboard/tensorboard_4b284b12bc4seed1/events.out.tfevents.1676597574.nid005516.63089.0 +3 -0
  30. tensorboard/tensorboard_4b284b12bc4seed1/events.out.tfevents.1676598062.nid005516.71283.0 +3 -0
  31. tensorboard/tensorboard_4b284b12bc4seed1/events.out.tfevents.1676598563.nid005516.76684.0 +3 -0
  32. tensorboard/tensorboard_4b284b12bc4seed1/events.out.tfevents.1676599066.nid005516.84582.0 +3 -0
  33. tensorboard/tensorboard_4b284b12bc4seed1/events.out.tfevents.1676599553.nid006969.122479.0 +3 -0
  34. tensorboard/tensorboard_4b284b12bc4seed1/events.out.tfevents.1676600044.nid006969.127475.0 +3 -0
  35. tensorboard/tensorboard_4b284b12bc4seed1/events.out.tfevents.1676600573.nid006969.5853.0 +3 -0
  36. tensorboard/tensorboard_4b284b12bc4seed1/events.out.tfevents.1677022444.nid006885.88040.0 +3 -0
  37. tensorboard/tensorboard_4b284b12bc4seed1/events.out.tfevents.1677022836.nid005287.20697.0 +3 -0
  38. tensorboard/tensorboard_4b284b12bc4seed1/events.out.tfevents.1677023221.nid005197.112586.0 +3 -0
  39. tensorboard/tensorboard_4b284b12bc4seed1/events.out.tfevents.1677023622.nid005332.86111.0 +3 -0
  40. tensorboard/tensorboard_4b284b12bc4seed1/events.out.tfevents.1677024036.nid006101.78835.0 +3 -0
  41. tensorboard/tensorboard_4b284b12bc4seed1/events.out.tfevents.1677024440.nid005197.4683.0 +3 -0
  42. tensorboard/tensorboard_4b284b12bc4seed1/events.out.tfevents.1677024845.nid005332.99971.0 +3 -0
  43. tensorboard/tensorboard_4b284b12bc4seed1/events.out.tfevents.1677025248.nid006885.2652.0 +3 -0
  44. tensorboard/tensorboard_4b284b12bc4seed1/events.out.tfevents.1677025666.nid006444.5012.0 +3 -0
  45. tensorboard/tensorboard_4b284b12bc4seed1/events.out.tfevents.1677026072.nid006885.11385.0 +3 -0
  46. tensorboard/tensorboard_4b284b12bc4seed1/events.out.tfevents.1677026470.nid006444.14023.0 +3 -0
  47. tensorboard/tensorboard_4b284b12bc4seed1/events.out.tfevents.1677026881.nid006101.114596.0 +3 -0
  48. tensorboard/tensorboard_4b284b12bc4seed1/events.out.tfevents.1677027297.nid005788.121184.0 +3 -0
  49. tensorboard/tensorboard_4b284b12bc4seed1/events.out.tfevents.1677027709.nid007155.46377.0 +3 -0
  50. tensorboard/tensorboard_4b284b12bc4seed1/events.out.tfevents.1677028112.nid007191.46020.0 +3 -0
sbatch_4b284b12bc4seed1.sh ADDED
@@ -0,0 +1,164 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #!/bin/bash
2
+ #SBATCH --exclude=nid007571,nid007112,nid006774,nid007502,nid007506,nid007507,nid005145,nid006692,nid007218,nid007123,nid006124,nid006123,nid007496,nid007237,nid006852,nid007206,nid006947,nid007212,nid006977,nid007222,nid005444,nid007219,nid007493,nid007221,nid005300,nid005619,nid006118,nid005203,nid006113,nid006481,nid007077,nid005208,nid005207,nid005879,nid005901
3
+ #SBATCH --nodes=32
4
+ #SBATCH --ntasks-per-node=1
5
+ #SBATCH --cpus-per-task=40
6
+ #SBATCH --mem=256G
7
+ #SBATCH -p standard-g
8
+ #SBATCH -t 48:00:00
9
+ #SBATCH --gpus-per-node=mi250:8
10
+ #SBATCH --exclusive=user
11
+ #SBATCH --hint=nomultithread
12
+ #SBATCH --account=project_462000119
13
+ #SBATCH -o logs/%j.out
14
+ #SBATCH -e logs/%j.err
15
+
16
+ VARIANT=4b284b12bc4seed1
17
+
18
+ # if run without sbatch, invoke here
19
+ if [ -z $SLURM_JOB_ID ]; then
20
+ mkdir -p logs
21
+ sbatch "$0"
22
+ exit
23
+ fi
24
+
25
+ set -euo pipefail
26
+
27
+ # symlink logs/latest.out and logs/latest.err
28
+ ln -f -s $SLURM_JOB_ID.out logs/latest.out
29
+ ln -f -s $SLURM_JOB_ID.err logs/latest.err
30
+
31
+ KILL_SWITCH_PATH=kill-switch-$VARIANT
32
+ CHECKPOINT_PATH=checkpoints_$VARIANT
33
+ TENSORBOARD_PATH=tensorboard_$VARIANT
34
+
35
+ # Data
36
+ VOCAB_FILE="gpt2/vocab.json"
37
+ MERGE_FILE="gpt2/merges.txt"
38
+
39
+ TRAIN_DATA_PATH=train12b.txt
40
+ # "train: 1.0 0:1 /scratch/project_462000119/data/c4_subsampled/gpt2tok_c4_en_12B_text_document"
41
+ VALID_DATA_PATH=val.txt
42
+ # "validation: 1.0 0:1 /scratch/project_462000119/data/c4_validation/gpt2tok_c4validation_rerun_text_document"
43
+
44
+ PP_SIZE=1
45
+ TP_SIZE=2
46
+
47
+ MICRO_BATCH_SIZE=2
48
+ GRADIENT_ACCUMULATION_STEPS=1
49
+ WORLD_SIZE=$((SLURM_GPUS_ON_NODE*SLURM_JOB_NUM_NODES))
50
+ GLOBAL_BATCH_SIZE=$((MICRO_BATCH_SIZE*WORLD_SIZE*GRADIENT_ACCUMULATION_STEPS))
51
+
52
+ # Model parameters
53
+ source model_params.sh
54
+ MODEL_PARAM=("${PARAM_4516M[@]}")
55
+ NHIDDEN=${MODEL_PARAM[0]}
56
+ FFN_HIDDEN_SIZE=${MODEL_PARAM[1]}
57
+ KV_SIZE=${MODEL_PARAM[2]}
58
+ NHEADS=${MODEL_PARAM[3]}
59
+ NLAYERS=${MODEL_PARAM[4]}
60
+ SEQ_LEN=2048
61
+
62
+ echo "Model parameters: d_model $NHIDDEN ffw_size $FFN_HIDDEN_SIZE kv_size $KV_SIZE n_heads $NHEADS n_layers $NLAYERS"
63
+
64
+ SAVE_INTERVAL=10000
65
+
66
+ # Tokens: 84_000_000_000
67
+ # -> Samples: 41_015_625.0
68
+ TRAIN_SAMPLES=41_015_625
69
+
70
+ OPTIMIZER_ARGS=" \
71
+ --optimizer adam \
72
+ --adam-beta1 0.9 \
73
+ --adam-beta2 0.95 \
74
+ --adam-eps 1e-8 \
75
+ --lr 2e-4 \
76
+ --min-lr 2e-5 \
77
+ --lr-decay-style cosine \
78
+ --lr-decay-samples $TRAIN_SAMPLES \
79
+ --lr-warmup-samples 410_156 \
80
+ --clip-grad 1.0 \
81
+ --weight-decay 1e-1 \
82
+ "
83
+
84
+ GPT_ARGS=" \
85
+ --num-layers $NLAYERS \
86
+ --hidden-size $NHIDDEN \
87
+ --num-attention-heads $NHEADS \
88
+ --kv-channels $KV_SIZE \
89
+ --ffn-hidden-size $FFN_HIDDEN_SIZE \
90
+ --seq-length $SEQ_LEN \
91
+ --max-position-embeddings $SEQ_LEN \
92
+ --micro-batch-size $MICRO_BATCH_SIZE \
93
+ --global-batch-size $GLOBAL_BATCH_SIZE \
94
+ --train-samples $TRAIN_SAMPLES \
95
+ --vocab-file $VOCAB_FILE \
96
+ --merge-file $MERGE_FILE \
97
+ --clip-grad 1.0 \
98
+ --kill-switch-path $KILL_SWITCH_PATH \
99
+ --bf16 \
100
+ $OPTIMIZER_ARGS \
101
+ "
102
+
103
+ OUTPUT_ARGS=" \
104
+ --log-interval 10 \
105
+ --save-interval $SAVE_INTERVAL \
106
+ --eval-interval 1000 \
107
+ --eval-iters 100 \
108
+ --tensorboard-dir $TENSORBOARD_PATH \
109
+ --tensorboard-queue-size 5 \
110
+ --log-timers-to-tensorboard \
111
+ --log-batch-size-to-tensorboard \
112
+ --log-validation-ppl-to-tensorboard \
113
+ "
114
+
115
+ ZERO_STAGE=0
116
+
117
+ mkdir -p ds_configs
118
+ DS_CONFIG_PATH="ds_configs/$SLURM_JOB_ID.json"
119
+
120
+ cat <<EOF > $DS_CONFIG_PATH
121
+ {
122
+ "train_micro_batch_size_per_gpu": $MICRO_BATCH_SIZE,
123
+ "train_batch_size": $GLOBAL_BATCH_SIZE,
124
+ "gradient_clipping": 1.0,
125
+ "zero_optimization": {
126
+ "stage": $ZERO_STAGE
127
+ },
128
+ "bf16": {
129
+ "enabled": true
130
+ },
131
+ "steps_per_print": 2000,
132
+ "wall_clock_breakdown": false
133
+ }
134
+ EOF
135
+
136
+ DEEPSPEED_ARGS=" \
137
+ --deepspeed \
138
+ --deepspeed_config $DS_CONFIG_PATH \
139
+ --zero-stage $ZERO_STAGE \
140
+ "
141
+
142
+ CMD=" \
143
+ Megatron-DeepSpeed/pretrain_gpt.py \
144
+ --tensor-model-parallel-size $TP_SIZE \
145
+ --pipeline-model-parallel-size $PP_SIZE \
146
+ $GPT_ARGS \
147
+ $OUTPUT_ARGS \
148
+ --save $CHECKPOINT_PATH \
149
+ --load $CHECKPOINT_PATH \
150
+ --train-weighted-split-paths-path $TRAIN_DATA_PATH \
151
+ --valid-weighted-split-paths-path $VALID_DATA_PATH \
152
+ --data-impl mmap \
153
+ --seed 1 \
154
+ $DEEPSPEED_ARGS \
155
+ "
156
+
157
+ echo $CMD
158
+
159
+ echo "START $SLURM_JOBID: $(date)"
160
+
161
+ # bash launch_srun.sh $CMD
162
+ srun --label launch.sh $CMD
163
+
164
+ echo "END $SLURM_JOBID: $(date)"
sbatch_4b284b12bc4seed2.sh ADDED
@@ -0,0 +1,164 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #!/bin/bash
2
+ #SBATCH --exclude=nid007571,nid007112,nid006774,nid007502,nid007506,nid007507,nid005145,nid006692,nid007218,nid007123,nid006124,nid006123,nid007496,nid007237,nid006852,nid007206,nid006947,nid007212,nid006977,nid007222,nid005444,nid007219,nid007493,nid007221,nid005300,nid005619,nid006118,nid005203,nid006113,nid006481,nid007077,nid005208,nid005207,nid005879,nid005901
3
+ #SBATCH --nodes=32
4
+ #SBATCH --ntasks-per-node=1
5
+ #SBATCH --cpus-per-task=40
6
+ #SBATCH --mem=256G
7
+ #SBATCH -p standard-g
8
+ #SBATCH -t 48:00:00
9
+ #SBATCH --gpus-per-node=mi250:8
10
+ #SBATCH --exclusive=user
11
+ #SBATCH --hint=nomultithread
12
+ #SBATCH --account=project_462000119
13
+ #SBATCH -o logs/%j.out
14
+ #SBATCH -e logs/%j.err
15
+
16
+ VARIANT=4b284b12bc4seed2
17
+
18
+ # if run without sbatch, invoke here
19
+ if [ -z $SLURM_JOB_ID ]; then
20
+ mkdir -p logs
21
+ sbatch "$0"
22
+ exit
23
+ fi
24
+
25
+ set -euo pipefail
26
+
27
+ # symlink logs/latest.out and logs/latest.err
28
+ ln -f -s $SLURM_JOB_ID.out logs/latest.out
29
+ ln -f -s $SLURM_JOB_ID.err logs/latest.err
30
+
31
+ KILL_SWITCH_PATH=kill-switch-$VARIANT
32
+ CHECKPOINT_PATH=checkpoints_$VARIANT
33
+ TENSORBOARD_PATH=tensorboard_$VARIANT
34
+
35
+ # Data
36
+ VOCAB_FILE="gpt2/vocab.json"
37
+ MERGE_FILE="gpt2/merges.txt"
38
+
39
+ TRAIN_DATA_PATH=train12b.txt
40
+ # "train: 1.0 0:1 /scratch/project_462000119/data/c4_subsampled/gpt2tok_c4_en_12B_text_document"
41
+ VALID_DATA_PATH=val.txt
42
+ # "validation: 1.0 0:1 /scratch/project_462000119/data/c4_validation/gpt2tok_c4validation_rerun_text_document"
43
+
44
+ PP_SIZE=1
45
+ TP_SIZE=2
46
+
47
+ MICRO_BATCH_SIZE=2
48
+ GRADIENT_ACCUMULATION_STEPS=1
49
+ WORLD_SIZE=$((SLURM_GPUS_ON_NODE*SLURM_JOB_NUM_NODES))
50
+ GLOBAL_BATCH_SIZE=$((MICRO_BATCH_SIZE*WORLD_SIZE*GRADIENT_ACCUMULATION_STEPS))
51
+
52
+ # Model parameters
53
+ source model_params.sh
54
+ MODEL_PARAM=("${PARAM_4516M[@]}")
55
+ NHIDDEN=${MODEL_PARAM[0]}
56
+ FFN_HIDDEN_SIZE=${MODEL_PARAM[1]}
57
+ KV_SIZE=${MODEL_PARAM[2]}
58
+ NHEADS=${MODEL_PARAM[3]}
59
+ NLAYERS=${MODEL_PARAM[4]}
60
+ SEQ_LEN=2048
61
+
62
+ echo "Model parameters: d_model $NHIDDEN ffw_size $FFN_HIDDEN_SIZE kv_size $KV_SIZE n_heads $NHEADS n_layers $NLAYERS"
63
+
64
+ SAVE_INTERVAL=10000
65
+
66
+ # Tokens: 84_000_000_000
67
+ # -> Samples: 41_015_625.0
68
+ TRAIN_SAMPLES=41_015_625
69
+
70
+ OPTIMIZER_ARGS=" \
71
+ --optimizer adam \
72
+ --adam-beta1 0.9 \
73
+ --adam-beta2 0.95 \
74
+ --adam-eps 1e-8 \
75
+ --lr 2e-4 \
76
+ --min-lr 2e-5 \
77
+ --lr-decay-style cosine \
78
+ --lr-decay-samples $TRAIN_SAMPLES \
79
+ --lr-warmup-samples 410_156 \
80
+ --clip-grad 1.0 \
81
+ --weight-decay 1e-1 \
82
+ "
83
+
84
+ GPT_ARGS=" \
85
+ --num-layers $NLAYERS \
86
+ --hidden-size $NHIDDEN \
87
+ --num-attention-heads $NHEADS \
88
+ --kv-channels $KV_SIZE \
89
+ --ffn-hidden-size $FFN_HIDDEN_SIZE \
90
+ --seq-length $SEQ_LEN \
91
+ --max-position-embeddings $SEQ_LEN \
92
+ --micro-batch-size $MICRO_BATCH_SIZE \
93
+ --global-batch-size $GLOBAL_BATCH_SIZE \
94
+ --train-samples $TRAIN_SAMPLES \
95
+ --vocab-file $VOCAB_FILE \
96
+ --merge-file $MERGE_FILE \
97
+ --clip-grad 1.0 \
98
+ --kill-switch-path $KILL_SWITCH_PATH \
99
+ --bf16 \
100
+ $OPTIMIZER_ARGS \
101
+ "
102
+
103
+ OUTPUT_ARGS=" \
104
+ --log-interval 10 \
105
+ --save-interval $SAVE_INTERVAL \
106
+ --eval-interval 1000 \
107
+ --eval-iters 100 \
108
+ --tensorboard-dir $TENSORBOARD_PATH \
109
+ --tensorboard-queue-size 5 \
110
+ --log-timers-to-tensorboard \
111
+ --log-batch-size-to-tensorboard \
112
+ --log-validation-ppl-to-tensorboard \
113
+ "
114
+
115
+ ZERO_STAGE=0
116
+
117
+ mkdir -p ds_configs
118
+ DS_CONFIG_PATH="ds_configs/$SLURM_JOB_ID.json"
119
+
120
+ cat <<EOF > $DS_CONFIG_PATH
121
+ {
122
+ "train_micro_batch_size_per_gpu": $MICRO_BATCH_SIZE,
123
+ "train_batch_size": $GLOBAL_BATCH_SIZE,
124
+ "gradient_clipping": 1.0,
125
+ "zero_optimization": {
126
+ "stage": $ZERO_STAGE
127
+ },
128
+ "bf16": {
129
+ "enabled": true
130
+ },
131
+ "steps_per_print": 2000,
132
+ "wall_clock_breakdown": false
133
+ }
134
+ EOF
135
+
136
+ DEEPSPEED_ARGS=" \
137
+ --deepspeed \
138
+ --deepspeed_config $DS_CONFIG_PATH \
139
+ --zero-stage $ZERO_STAGE \
140
+ "
141
+
142
+ CMD=" \
143
+ Megatron-DeepSpeed/pretrain_gpt.py \
144
+ --tensor-model-parallel-size $TP_SIZE \
145
+ --pipeline-model-parallel-size $PP_SIZE \
146
+ $GPT_ARGS \
147
+ $OUTPUT_ARGS \
148
+ --save $CHECKPOINT_PATH \
149
+ --load $CHECKPOINT_PATH \
150
+ --train-weighted-split-paths-path $TRAIN_DATA_PATH \
151
+ --valid-weighted-split-paths-path $VALID_DATA_PATH \
152
+ --data-impl mmap \
153
+ --seed 2 \
154
+ $DEEPSPEED_ARGS \
155
+ "
156
+
157
+ echo $CMD
158
+
159
+ echo "START $SLURM_JOBID: $(date)"
160
+
161
+ # bash launch_srun.sh $CMD
162
+ srun --label launch.sh $CMD
163
+
164
+ echo "END $SLURM_JOBID: $(date)"
sbatch_4b284b12bc4seed3.sh ADDED
@@ -0,0 +1,164 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #!/bin/bash
2
+ #SBATCH --exclude=nid007571,nid007112,nid006774,nid007502,nid007506,nid007507,nid005145,nid006692,nid007218,nid007123,nid006124,nid006123,nid007496,nid007237,nid006852,nid007206,nid006947,nid007212,nid006977,nid007222,nid005444,nid007219,nid007493,nid007221,nid005300,nid005619,nid006118,nid005203,nid006113,nid006481,nid007077,nid005208,nid005207,nid005879,nid005901
3
+ #SBATCH --nodes=32
4
+ #SBATCH --ntasks-per-node=1
5
+ #SBATCH --cpus-per-task=40
6
+ #SBATCH --mem=256G
7
+ #SBATCH -p standard-g
8
+ #SBATCH -t 48:00:00
9
+ #SBATCH --gpus-per-node=mi250:8
10
+ #SBATCH --exclusive=user
11
+ #SBATCH --hint=nomultithread
12
+ #SBATCH --account=project_462000119
13
+ #SBATCH -o logs/%j.out
14
+ #SBATCH -e logs/%j.err
15
+
16
+ VARIANT=4b284b12bc4seed3
17
+
18
+ # if run without sbatch, invoke here
19
+ if [ -z $SLURM_JOB_ID ]; then
20
+ mkdir -p logs
21
+ sbatch "$0"
22
+ exit
23
+ fi
24
+
25
+ set -euo pipefail
26
+
27
+ # symlink logs/latest.out and logs/latest.err
28
+ ln -f -s $SLURM_JOB_ID.out logs/latest.out
29
+ ln -f -s $SLURM_JOB_ID.err logs/latest.err
30
+
31
+ KILL_SWITCH_PATH=kill-switch-$VARIANT
32
+ CHECKPOINT_PATH=checkpoints_$VARIANT
33
+ TENSORBOARD_PATH=tensorboard_$VARIANT
34
+
35
+ # Data
36
+ VOCAB_FILE="gpt2/vocab.json"
37
+ MERGE_FILE="gpt2/merges.txt"
38
+
39
+ TRAIN_DATA_PATH=train12b.txt
40
+ # "train: 1.0 0:1 /scratch/project_462000119/data/c4_subsampled/gpt2tok_c4_en_12B_text_document"
41
+ VALID_DATA_PATH=val.txt
42
+ # "validation: 1.0 0:1 /scratch/project_462000119/data/c4_validation/gpt2tok_c4validation_rerun_text_document"
43
+
44
+ PP_SIZE=1
45
+ TP_SIZE=2
46
+
47
+ MICRO_BATCH_SIZE=2
48
+ GRADIENT_ACCUMULATION_STEPS=1
49
+ WORLD_SIZE=$((SLURM_GPUS_ON_NODE*SLURM_JOB_NUM_NODES))
50
+ GLOBAL_BATCH_SIZE=$((MICRO_BATCH_SIZE*WORLD_SIZE*GRADIENT_ACCUMULATION_STEPS))
51
+
52
+ # Model parameters
53
+ source model_params.sh
54
+ MODEL_PARAM=("${PARAM_4516M[@]}")
55
+ NHIDDEN=${MODEL_PARAM[0]}
56
+ FFN_HIDDEN_SIZE=${MODEL_PARAM[1]}
57
+ KV_SIZE=${MODEL_PARAM[2]}
58
+ NHEADS=${MODEL_PARAM[3]}
59
+ NLAYERS=${MODEL_PARAM[4]}
60
+ SEQ_LEN=2048
61
+
62
+ echo "Model parameters: d_model $NHIDDEN ffw_size $FFN_HIDDEN_SIZE kv_size $KV_SIZE n_heads $NHEADS n_layers $NLAYERS"
63
+
64
+ SAVE_INTERVAL=10000
65
+
66
+ # Tokens: 84_000_000_000
67
+ # -> Samples: 41_015_625.0
68
+ TRAIN_SAMPLES=41_015_625
69
+
70
+ OPTIMIZER_ARGS=" \
71
+ --optimizer adam \
72
+ --adam-beta1 0.9 \
73
+ --adam-beta2 0.95 \
74
+ --adam-eps 1e-8 \
75
+ --lr 2e-4 \
76
+ --min-lr 2e-5 \
77
+ --lr-decay-style cosine \
78
+ --lr-decay-samples $TRAIN_SAMPLES \
79
+ --lr-warmup-samples 410_156 \
80
+ --clip-grad 1.0 \
81
+ --weight-decay 1e-1 \
82
+ "
83
+
84
+ GPT_ARGS=" \
85
+ --num-layers $NLAYERS \
86
+ --hidden-size $NHIDDEN \
87
+ --num-attention-heads $NHEADS \
88
+ --kv-channels $KV_SIZE \
89
+ --ffn-hidden-size $FFN_HIDDEN_SIZE \
90
+ --seq-length $SEQ_LEN \
91
+ --max-position-embeddings $SEQ_LEN \
92
+ --micro-batch-size $MICRO_BATCH_SIZE \
93
+ --global-batch-size $GLOBAL_BATCH_SIZE \
94
+ --train-samples $TRAIN_SAMPLES \
95
+ --vocab-file $VOCAB_FILE \
96
+ --merge-file $MERGE_FILE \
97
+ --clip-grad 1.0 \
98
+ --kill-switch-path $KILL_SWITCH_PATH \
99
+ --bf16 \
100
+ $OPTIMIZER_ARGS \
101
+ "
102
+
103
+ OUTPUT_ARGS=" \
104
+ --log-interval 10 \
105
+ --save-interval $SAVE_INTERVAL \
106
+ --eval-interval 1000 \
107
+ --eval-iters 100 \
108
+ --tensorboard-dir $TENSORBOARD_PATH \
109
+ --tensorboard-queue-size 5 \
110
+ --log-timers-to-tensorboard \
111
+ --log-batch-size-to-tensorboard \
112
+ --log-validation-ppl-to-tensorboard \
113
+ "
114
+
115
+ ZERO_STAGE=0
116
+
117
+ mkdir -p ds_configs
118
+ DS_CONFIG_PATH="ds_configs/$SLURM_JOB_ID.json"
119
+
120
+ cat <<EOF > $DS_CONFIG_PATH
121
+ {
122
+ "train_micro_batch_size_per_gpu": $MICRO_BATCH_SIZE,
123
+ "train_batch_size": $GLOBAL_BATCH_SIZE,
124
+ "gradient_clipping": 1.0,
125
+ "zero_optimization": {
126
+ "stage": $ZERO_STAGE
127
+ },
128
+ "bf16": {
129
+ "enabled": true
130
+ },
131
+ "steps_per_print": 2000,
132
+ "wall_clock_breakdown": false
133
+ }
134
+ EOF
135
+
136
+ DEEPSPEED_ARGS=" \
137
+ --deepspeed \
138
+ --deepspeed_config $DS_CONFIG_PATH \
139
+ --zero-stage $ZERO_STAGE \
140
+ "
141
+
142
+ CMD=" \
143
+ Megatron-DeepSpeed/pretrain_gpt.py \
144
+ --tensor-model-parallel-size $TP_SIZE \
145
+ --pipeline-model-parallel-size $PP_SIZE \
146
+ $GPT_ARGS \
147
+ $OUTPUT_ARGS \
148
+ --save $CHECKPOINT_PATH \
149
+ --load $CHECKPOINT_PATH \
150
+ --train-weighted-split-paths-path $TRAIN_DATA_PATH \
151
+ --valid-weighted-split-paths-path $VALID_DATA_PATH \
152
+ --data-impl mmap \
153
+ --seed 3 \
154
+ $DEEPSPEED_ARGS \
155
+ "
156
+
157
+ echo $CMD
158
+
159
+ echo "START $SLURM_JOBID: $(date)"
160
+
161
+ # bash launch_srun.sh $CMD
162
+ srun --label launch.sh $CMD
163
+
164
+ echo "END $SLURM_JOBID: $(date)"
sbatch_4b284b12bc4seed4.sh ADDED
@@ -0,0 +1,164 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #!/bin/bash
2
+ #SBATCH --exclude=nid007571,nid007112,nid006774,nid007502,nid007506,nid007507,nid005145,nid006692,nid007218,nid007123,nid006124,nid006123,nid007496,nid007237,nid006852,nid007206,nid006947,nid007212,nid006977,nid007222,nid005444,nid007219,nid007493,nid007221,nid005300,nid005619,nid006118,nid005203,nid006113,nid006481,nid007077,nid005208,nid005207,nid005879,nid005901
3
+ #SBATCH --nodes=32
4
+ #SBATCH --ntasks-per-node=1
5
+ #SBATCH --cpus-per-task=40
6
+ #SBATCH --mem=256G
7
+ #SBATCH -p standard-g
8
+ #SBATCH -t 48:00:00
9
+ #SBATCH --gpus-per-node=mi250:8
10
+ #SBATCH --exclusive=user
11
+ #SBATCH --hint=nomultithread
12
+ #SBATCH --account=project_462000119
13
+ #SBATCH -o logs/%j.out
14
+ #SBATCH -e logs/%j.err
15
+
16
+ VARIANT=4b284b12bc4seed4
17
+
18
+ # if run without sbatch, invoke here
19
+ if [ -z $SLURM_JOB_ID ]; then
20
+ mkdir -p logs
21
+ sbatch "$0"
22
+ exit
23
+ fi
24
+
25
+ set -euo pipefail
26
+
27
+ # symlink logs/latest.out and logs/latest.err
28
+ ln -f -s $SLURM_JOB_ID.out logs/latest.out
29
+ ln -f -s $SLURM_JOB_ID.err logs/latest.err
30
+
31
+ KILL_SWITCH_PATH=kill-switch-$VARIANT
32
+ CHECKPOINT_PATH=checkpoints_$VARIANT
33
+ TENSORBOARD_PATH=tensorboard_$VARIANT
34
+
35
+ # Data
36
+ VOCAB_FILE="gpt2/vocab.json"
37
+ MERGE_FILE="gpt2/merges.txt"
38
+
39
+ TRAIN_DATA_PATH=train12b.txt
40
+ # "train: 1.0 0:1 /scratch/project_462000119/data/c4_subsampled/gpt2tok_c4_en_12B_text_document"
41
+ VALID_DATA_PATH=val.txt
42
+ # "validation: 1.0 0:1 /scratch/project_462000119/data/c4_validation/gpt2tok_c4validation_rerun_text_document"
43
+
44
+ PP_SIZE=1
45
+ TP_SIZE=2
46
+
47
+ MICRO_BATCH_SIZE=2
48
+ GRADIENT_ACCUMULATION_STEPS=1
49
+ WORLD_SIZE=$((SLURM_GPUS_ON_NODE*SLURM_JOB_NUM_NODES))
50
+ GLOBAL_BATCH_SIZE=$((MICRO_BATCH_SIZE*WORLD_SIZE*GRADIENT_ACCUMULATION_STEPS))
51
+
52
+ # Model parameters
53
+ source model_params.sh
54
+ MODEL_PARAM=("${PARAM_4516M[@]}")
55
+ NHIDDEN=${MODEL_PARAM[0]}
56
+ FFN_HIDDEN_SIZE=${MODEL_PARAM[1]}
57
+ KV_SIZE=${MODEL_PARAM[2]}
58
+ NHEADS=${MODEL_PARAM[3]}
59
+ NLAYERS=${MODEL_PARAM[4]}
60
+ SEQ_LEN=2048
61
+
62
+ echo "Model parameters: d_model $NHIDDEN ffw_size $FFN_HIDDEN_SIZE kv_size $KV_SIZE n_heads $NHEADS n_layers $NLAYERS"
63
+
64
+ SAVE_INTERVAL=10000
65
+
66
+ # Tokens: 84_000_000_000
67
+ # -> Samples: 41_015_625.0
68
+ TRAIN_SAMPLES=41_015_625
69
+
70
+ OPTIMIZER_ARGS=" \
71
+ --optimizer adam \
72
+ --adam-beta1 0.9 \
73
+ --adam-beta2 0.95 \
74
+ --adam-eps 1e-8 \
75
+ --lr 2e-4 \
76
+ --min-lr 2e-5 \
77
+ --lr-decay-style cosine \
78
+ --lr-decay-samples $TRAIN_SAMPLES \
79
+ --lr-warmup-samples 410_156 \
80
+ --clip-grad 1.0 \
81
+ --weight-decay 1e-1 \
82
+ "
83
+
84
+ GPT_ARGS=" \
85
+ --num-layers $NLAYERS \
86
+ --hidden-size $NHIDDEN \
87
+ --num-attention-heads $NHEADS \
88
+ --kv-channels $KV_SIZE \
89
+ --ffn-hidden-size $FFN_HIDDEN_SIZE \
90
+ --seq-length $SEQ_LEN \
91
+ --max-position-embeddings $SEQ_LEN \
92
+ --micro-batch-size $MICRO_BATCH_SIZE \
93
+ --global-batch-size $GLOBAL_BATCH_SIZE \
94
+ --train-samples $TRAIN_SAMPLES \
95
+ --vocab-file $VOCAB_FILE \
96
+ --merge-file $MERGE_FILE \
97
+ --clip-grad 1.0 \
98
+ --kill-switch-path $KILL_SWITCH_PATH \
99
+ --bf16 \
100
+ $OPTIMIZER_ARGS \
101
+ "
102
+
103
+ OUTPUT_ARGS=" \
104
+ --log-interval 10 \
105
+ --save-interval $SAVE_INTERVAL \
106
+ --eval-interval 1000 \
107
+ --eval-iters 100 \
108
+ --tensorboard-dir $TENSORBOARD_PATH \
109
+ --tensorboard-queue-size 5 \
110
+ --log-timers-to-tensorboard \
111
+ --log-batch-size-to-tensorboard \
112
+ --log-validation-ppl-to-tensorboard \
113
+ "
114
+
115
+ ZERO_STAGE=0
116
+
117
+ mkdir -p ds_configs
118
+ DS_CONFIG_PATH="ds_configs/$SLURM_JOB_ID.json"
119
+
120
+ cat <<EOF > $DS_CONFIG_PATH
121
+ {
122
+ "train_micro_batch_size_per_gpu": $MICRO_BATCH_SIZE,
123
+ "train_batch_size": $GLOBAL_BATCH_SIZE,
124
+ "gradient_clipping": 1.0,
125
+ "zero_optimization": {
126
+ "stage": $ZERO_STAGE
127
+ },
128
+ "bf16": {
129
+ "enabled": true
130
+ },
131
+ "steps_per_print": 2000,
132
+ "wall_clock_breakdown": false
133
+ }
134
+ EOF
135
+
136
+ DEEPSPEED_ARGS=" \
137
+ --deepspeed \
138
+ --deepspeed_config $DS_CONFIG_PATH \
139
+ --zero-stage $ZERO_STAGE \
140
+ "
141
+
142
+ CMD=" \
143
+ Megatron-DeepSpeed/pretrain_gpt.py \
144
+ --tensor-model-parallel-size $TP_SIZE \
145
+ --pipeline-model-parallel-size $PP_SIZE \
146
+ $GPT_ARGS \
147
+ $OUTPUT_ARGS \
148
+ --save $CHECKPOINT_PATH \
149
+ --load $CHECKPOINT_PATH \
150
+ --train-weighted-split-paths-path $TRAIN_DATA_PATH \
151
+ --valid-weighted-split-paths-path $VALID_DATA_PATH \
152
+ --data-impl mmap \
153
+ --seed 4 \
154
+ $DEEPSPEED_ARGS \
155
+ "
156
+
157
+ echo $CMD
158
+
159
+ echo "START $SLURM_JOBID: $(date)"
160
+
161
+ # bash launch_srun.sh $CMD
162
+ srun --label launch.sh $CMD
163
+
164
+ echo "END $SLURM_JOBID: $(date)"
sbatch_4b284b17bc4seed1.sh ADDED
@@ -0,0 +1,164 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #!/bin/bash
2
+ #SBATCH --exclude=nid007571,nid007112,nid006774,nid007502,nid007506,nid007507,nid005145,nid006692,nid007218,nid007123,nid006124,nid006123,nid007496,nid007237,nid006852,nid007206,nid006947,nid007212,nid006977,nid007222,nid005444,nid007219,nid007493,nid007221,nid005300,nid005619,nid006118,nid005203,nid006113,nid006481,nid007077,nid005208,nid005207,nid005879,nid005901
3
+ #SBATCH --nodes=32
4
+ #SBATCH --ntasks-per-node=1
5
+ #SBATCH --cpus-per-task=40
6
+ #SBATCH --mem=256G
7
+ #SBATCH -p standard-g
8
+ #SBATCH -t 48:00:00
9
+ #SBATCH --gpus-per-node=mi250:8
10
+ #SBATCH --exclusive=user
11
+ #SBATCH --hint=nomultithread
12
+ #SBATCH --account=project_462000119
13
+ #SBATCH -o logs/%j.out
14
+ #SBATCH -e logs/%j.err
15
+
16
+ VARIANT=4b284b17bc4seed1
17
+
18
+ # if run without sbatch, invoke here
19
+ if [ -z $SLURM_JOB_ID ]; then
20
+ mkdir -p logs
21
+ sbatch "$0"
22
+ exit
23
+ fi
24
+
25
+ set -euo pipefail
26
+
27
+ # symlink logs/latest.out and logs/latest.err
28
+ ln -f -s $SLURM_JOB_ID.out logs/latest.out
29
+ ln -f -s $SLURM_JOB_ID.err logs/latest.err
30
+
31
+ KILL_SWITCH_PATH=kill-switch-$VARIANT
32
+ CHECKPOINT_PATH=checkpoints_$VARIANT
33
+ TENSORBOARD_PATH=tensorboard_$VARIANT
34
+
35
+ # Data
36
+ VOCAB_FILE="gpt2/vocab.json"
37
+ MERGE_FILE="gpt2/merges.txt"
38
+
39
+ TRAIN_DATA_PATH=train17b.txt
40
+ # "train: 1.0 0:1 /scratch/project_462000119/data/c4_subsampled/gpt2tok_c4_en_17B_text_document"
41
+ VALID_DATA_PATH=val.txt
42
+ # "validation: 1.0 0:1 /scratch/project_462000119/data/c4_validation/gpt2tok_c4validation_rerun_text_document"
43
+
44
+ PP_SIZE=1
45
+ TP_SIZE=2
46
+
47
+ MICRO_BATCH_SIZE=2
48
+ GRADIENT_ACCUMULATION_STEPS=1
49
+ WORLD_SIZE=$((SLURM_GPUS_ON_NODE*SLURM_JOB_NUM_NODES))
50
+ GLOBAL_BATCH_SIZE=$((MICRO_BATCH_SIZE*WORLD_SIZE*GRADIENT_ACCUMULATION_STEPS))
51
+
52
+ # Model parameters
53
+ source model_params.sh
54
+ MODEL_PARAM=("${PARAM_4516M[@]}")
55
+ NHIDDEN=${MODEL_PARAM[0]}
56
+ FFN_HIDDEN_SIZE=${MODEL_PARAM[1]}
57
+ KV_SIZE=${MODEL_PARAM[2]}
58
+ NHEADS=${MODEL_PARAM[3]}
59
+ NLAYERS=${MODEL_PARAM[4]}
60
+ SEQ_LEN=2048
61
+
62
+ echo "Model parameters: d_model $NHIDDEN ffw_size $FFN_HIDDEN_SIZE kv_size $KV_SIZE n_heads $NHEADS n_layers $NLAYERS"
63
+
64
+ SAVE_INTERVAL=10000
65
+
66
+ # Tokens: 84_000_000_000
67
+ # -> Samples: 41_015_625.0
68
+ TRAIN_SAMPLES=41_015_625
69
+
70
+ OPTIMIZER_ARGS=" \
71
+ --optimizer adam \
72
+ --adam-beta1 0.9 \
73
+ --adam-beta2 0.95 \
74
+ --adam-eps 1e-8 \
75
+ --lr 2e-4 \
76
+ --min-lr 2e-5 \
77
+ --lr-decay-style cosine \
78
+ --lr-decay-samples $TRAIN_SAMPLES \
79
+ --lr-warmup-samples 410_156 \
80
+ --clip-grad 1.0 \
81
+ --weight-decay 1e-1 \
82
+ "
83
+
84
+ GPT_ARGS=" \
85
+ --num-layers $NLAYERS \
86
+ --hidden-size $NHIDDEN \
87
+ --num-attention-heads $NHEADS \
88
+ --kv-channels $KV_SIZE \
89
+ --ffn-hidden-size $FFN_HIDDEN_SIZE \
90
+ --seq-length $SEQ_LEN \
91
+ --max-position-embeddings $SEQ_LEN \
92
+ --micro-batch-size $MICRO_BATCH_SIZE \
93
+ --global-batch-size $GLOBAL_BATCH_SIZE \
94
+ --train-samples $TRAIN_SAMPLES \
95
+ --vocab-file $VOCAB_FILE \
96
+ --merge-file $MERGE_FILE \
97
+ --clip-grad 1.0 \
98
+ --kill-switch-path $KILL_SWITCH_PATH \
99
+ --bf16 \
100
+ $OPTIMIZER_ARGS \
101
+ "
102
+
103
+ OUTPUT_ARGS=" \
104
+ --log-interval 10 \
105
+ --save-interval $SAVE_INTERVAL \
106
+ --eval-interval 1000 \
107
+ --eval-iters 100 \
108
+ --tensorboard-dir $TENSORBOARD_PATH \
109
+ --tensorboard-queue-size 5 \
110
+ --log-timers-to-tensorboard \
111
+ --log-batch-size-to-tensorboard \
112
+ --log-validation-ppl-to-tensorboard \
113
+ "
114
+
115
+ ZERO_STAGE=0
116
+
117
+ mkdir -p ds_configs
118
+ DS_CONFIG_PATH="ds_configs/$SLURM_JOB_ID.json"
119
+
120
+ cat <<EOF > $DS_CONFIG_PATH
121
+ {
122
+ "train_micro_batch_size_per_gpu": $MICRO_BATCH_SIZE,
123
+ "train_batch_size": $GLOBAL_BATCH_SIZE,
124
+ "gradient_clipping": 1.0,
125
+ "zero_optimization": {
126
+ "stage": $ZERO_STAGE
127
+ },
128
+ "bf16": {
129
+ "enabled": true
130
+ },
131
+ "steps_per_print": 2000,
132
+ "wall_clock_breakdown": false
133
+ }
134
+ EOF
135
+
136
+ DEEPSPEED_ARGS=" \
137
+ --deepspeed \
138
+ --deepspeed_config $DS_CONFIG_PATH \
139
+ --zero-stage $ZERO_STAGE \
140
+ "
141
+
142
+ CMD=" \
143
+ Megatron-DeepSpeed/pretrain_gpt.py \
144
+ --tensor-model-parallel-size $TP_SIZE \
145
+ --pipeline-model-parallel-size $PP_SIZE \
146
+ $GPT_ARGS \
147
+ $OUTPUT_ARGS \
148
+ --save $CHECKPOINT_PATH \
149
+ --load $CHECKPOINT_PATH \
150
+ --train-weighted-split-paths-path $TRAIN_DATA_PATH \
151
+ --valid-weighted-split-paths-path $VALID_DATA_PATH \
152
+ --data-impl mmap \
153
+ --seed 1 \
154
+ $DEEPSPEED_ARGS \
155
+ "
156
+
157
+ echo $CMD
158
+
159
+ echo "START $SLURM_JOBID: $(date)"
160
+
161
+ # bash launch_srun.sh $CMD
162
+ srun --label launch.sh $CMD
163
+
164
+ echo "END $SLURM_JOBID: $(date)"
sbatch_4b284b17bc4seed2.sh ADDED
@@ -0,0 +1,164 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #!/bin/bash
2
+ #SBATCH --exclude=nid007571,nid007112,nid006774,nid007502,nid007506,nid007507,nid005145,nid006692,nid007218,nid007123,nid006124,nid006123,nid007496,nid007237,nid006852,nid007206,nid006947,nid007212,nid006977,nid007222,nid005444,nid007219,nid007493,nid007221,nid005300,nid005619,nid006118,nid005203,nid006113,nid006481,nid007077,nid005208,nid005207,nid005879,nid005901
3
+ #SBATCH --nodes=32
4
+ #SBATCH --ntasks-per-node=1
5
+ #SBATCH --cpus-per-task=40
6
+ #SBATCH --mem=256G
7
+ #SBATCH -p standard-g
8
+ #SBATCH -t 48:00:00
9
+ #SBATCH --gpus-per-node=mi250:8
10
+ #SBATCH --exclusive=user
11
+ #SBATCH --hint=nomultithread
12
+ #SBATCH --account=project_462000119
13
+ #SBATCH -o logs/%j.out
14
+ #SBATCH -e logs/%j.err
15
+
16
+ VARIANT=4b284b17bc4seed2
17
+
18
+ # if run without sbatch, invoke here
19
+ if [ -z $SLURM_JOB_ID ]; then
20
+ mkdir -p logs
21
+ sbatch "$0"
22
+ exit
23
+ fi
24
+
25
+ set -euo pipefail
26
+
27
+ # symlink logs/latest.out and logs/latest.err
28
+ ln -f -s $SLURM_JOB_ID.out logs/latest.out
29
+ ln -f -s $SLURM_JOB_ID.err logs/latest.err
30
+
31
+ KILL_SWITCH_PATH=kill-switch-$VARIANT
32
+ CHECKPOINT_PATH=checkpoints_$VARIANT
33
+ TENSORBOARD_PATH=tensorboard_$VARIANT
34
+
35
+ # Data
36
+ VOCAB_FILE="gpt2/vocab.json"
37
+ MERGE_FILE="gpt2/merges.txt"
38
+
39
+ TRAIN_DATA_PATH=train17b.txt
40
+ # "train: 1.0 0:1 /scratch/project_462000119/data/c4_subsampled/gpt2tok_c4_en_17B_text_document"
41
+ VALID_DATA_PATH=val.txt
42
+ # "validation: 1.0 0:1 /scratch/project_462000119/data/c4_validation/gpt2tok_c4validation_rerun_text_document"
43
+
44
+ PP_SIZE=1
45
+ TP_SIZE=2
46
+
47
+ MICRO_BATCH_SIZE=2
48
+ GRADIENT_ACCUMULATION_STEPS=1
49
+ WORLD_SIZE=$((SLURM_GPUS_ON_NODE*SLURM_JOB_NUM_NODES))
50
+ GLOBAL_BATCH_SIZE=$((MICRO_BATCH_SIZE*WORLD_SIZE*GRADIENT_ACCUMULATION_STEPS))
51
+
52
+ # Model parameters
53
+ source model_params.sh
54
+ MODEL_PARAM=("${PARAM_4516M[@]}")
55
+ NHIDDEN=${MODEL_PARAM[0]}
56
+ FFN_HIDDEN_SIZE=${MODEL_PARAM[1]}
57
+ KV_SIZE=${MODEL_PARAM[2]}
58
+ NHEADS=${MODEL_PARAM[3]}
59
+ NLAYERS=${MODEL_PARAM[4]}
60
+ SEQ_LEN=2048
61
+
62
+ echo "Model parameters: d_model $NHIDDEN ffw_size $FFN_HIDDEN_SIZE kv_size $KV_SIZE n_heads $NHEADS n_layers $NLAYERS"
63
+
64
+ SAVE_INTERVAL=10000
65
+
66
+ # Tokens: 84_000_000_000
67
+ # -> Samples: 41_015_625.0
68
+ TRAIN_SAMPLES=41_015_625
69
+
70
+ OPTIMIZER_ARGS=" \
71
+ --optimizer adam \
72
+ --adam-beta1 0.9 \
73
+ --adam-beta2 0.95 \
74
+ --adam-eps 1e-8 \
75
+ --lr 2e-4 \
76
+ --min-lr 2e-5 \
77
+ --lr-decay-style cosine \
78
+ --lr-decay-samples $TRAIN_SAMPLES \
79
+ --lr-warmup-samples 410_156 \
80
+ --clip-grad 1.0 \
81
+ --weight-decay 1e-1 \
82
+ "
83
+
84
+ GPT_ARGS=" \
85
+ --num-layers $NLAYERS \
86
+ --hidden-size $NHIDDEN \
87
+ --num-attention-heads $NHEADS \
88
+ --kv-channels $KV_SIZE \
89
+ --ffn-hidden-size $FFN_HIDDEN_SIZE \
90
+ --seq-length $SEQ_LEN \
91
+ --max-position-embeddings $SEQ_LEN \
92
+ --micro-batch-size $MICRO_BATCH_SIZE \
93
+ --global-batch-size $GLOBAL_BATCH_SIZE \
94
+ --train-samples $TRAIN_SAMPLES \
95
+ --vocab-file $VOCAB_FILE \
96
+ --merge-file $MERGE_FILE \
97
+ --clip-grad 1.0 \
98
+ --kill-switch-path $KILL_SWITCH_PATH \
99
+ --bf16 \
100
+ $OPTIMIZER_ARGS \
101
+ "
102
+
103
+ OUTPUT_ARGS=" \
104
+ --log-interval 10 \
105
+ --save-interval $SAVE_INTERVAL \
106
+ --eval-interval 1000 \
107
+ --eval-iters 100 \
108
+ --tensorboard-dir $TENSORBOARD_PATH \
109
+ --tensorboard-queue-size 5 \
110
+ --log-timers-to-tensorboard \
111
+ --log-batch-size-to-tensorboard \
112
+ --log-validation-ppl-to-tensorboard \
113
+ "
114
+
115
+ ZERO_STAGE=0
116
+
117
+ mkdir -p ds_configs
118
+ DS_CONFIG_PATH="ds_configs/$SLURM_JOB_ID.json"
119
+
120
+ cat <<EOF > $DS_CONFIG_PATH
121
+ {
122
+ "train_micro_batch_size_per_gpu": $MICRO_BATCH_SIZE,
123
+ "train_batch_size": $GLOBAL_BATCH_SIZE,
124
+ "gradient_clipping": 1.0,
125
+ "zero_optimization": {
126
+ "stage": $ZERO_STAGE
127
+ },
128
+ "bf16": {
129
+ "enabled": true
130
+ },
131
+ "steps_per_print": 2000,
132
+ "wall_clock_breakdown": false
133
+ }
134
+ EOF
135
+
136
+ DEEPSPEED_ARGS=" \
137
+ --deepspeed \
138
+ --deepspeed_config $DS_CONFIG_PATH \
139
+ --zero-stage $ZERO_STAGE \
140
+ "
141
+
142
+ CMD=" \
143
+ Megatron-DeepSpeed/pretrain_gpt.py \
144
+ --tensor-model-parallel-size $TP_SIZE \
145
+ --pipeline-model-parallel-size $PP_SIZE \
146
+ $GPT_ARGS \
147
+ $OUTPUT_ARGS \
148
+ --save $CHECKPOINT_PATH \
149
+ --load $CHECKPOINT_PATH \
150
+ --train-weighted-split-paths-path $TRAIN_DATA_PATH \
151
+ --valid-weighted-split-paths-path $VALID_DATA_PATH \
152
+ --data-impl mmap \
153
+ --seed 2 \
154
+ $DEEPSPEED_ARGS \
155
+ "
156
+
157
+ echo $CMD
158
+
159
+ echo "START $SLURM_JOBID: $(date)"
160
+
161
+ # bash launch_srun.sh $CMD
162
+ srun --label launch.sh $CMD
163
+
164
+ echo "END $SLURM_JOBID: $(date)"
sbatch_4b284b17bc4seed3.sh ADDED
@@ -0,0 +1,164 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #!/bin/bash
2
+ #SBATCH --exclude=nid007571,nid007112,nid006774,nid007502,nid007506,nid007507,nid005145,nid006692,nid007218,nid007123,nid006124,nid006123,nid007496,nid007237,nid006852,nid007206,nid006947,nid007212,nid006977,nid007222,nid005444,nid007219,nid007493,nid007221,nid005300,nid005619,nid006118,nid005203,nid006113,nid006481,nid007077,nid005208,nid005207,nid005879,nid005901
3
+ #SBATCH --nodes=32
4
+ #SBATCH --ntasks-per-node=1
5
+ #SBATCH --cpus-per-task=40
6
+ #SBATCH --mem=256G
7
+ #SBATCH -p standard-g
8
+ #SBATCH -t 48:00:00
9
+ #SBATCH --gpus-per-node=mi250:8
10
+ #SBATCH --exclusive=user
11
+ #SBATCH --hint=nomultithread
12
+ #SBATCH --account=project_462000119
13
+ #SBATCH -o logs/%j.out
14
+ #SBATCH -e logs/%j.err
15
+
16
+ VARIANT=4b284b17bc4seed3
17
+
18
+ # if run without sbatch, invoke here
19
+ if [ -z $SLURM_JOB_ID ]; then
20
+ mkdir -p logs
21
+ sbatch "$0"
22
+ exit
23
+ fi
24
+
25
+ set -euo pipefail
26
+
27
+ # symlink logs/latest.out and logs/latest.err
28
+ ln -f -s $SLURM_JOB_ID.out logs/latest.out
29
+ ln -f -s $SLURM_JOB_ID.err logs/latest.err
30
+
31
+ KILL_SWITCH_PATH=kill-switch-$VARIANT
32
+ CHECKPOINT_PATH=checkpoints_$VARIANT
33
+ TENSORBOARD_PATH=tensorboard_$VARIANT
34
+
35
+ # Data
36
+ VOCAB_FILE="gpt2/vocab.json"
37
+ MERGE_FILE="gpt2/merges.txt"
38
+
39
+ TRAIN_DATA_PATH=train17b.txt
40
+ # "train: 1.0 0:1 /scratch/project_462000119/data/c4_subsampled/gpt2tok_c4_en_17B_text_document"
41
+ VALID_DATA_PATH=val.txt
42
+ # "validation: 1.0 0:1 /scratch/project_462000119/data/c4_validation/gpt2tok_c4validation_rerun_text_document"
43
+
44
+ PP_SIZE=1
45
+ TP_SIZE=2
46
+
47
+ MICRO_BATCH_SIZE=2
48
+ GRADIENT_ACCUMULATION_STEPS=1
49
+ WORLD_SIZE=$((SLURM_GPUS_ON_NODE*SLURM_JOB_NUM_NODES))
50
+ GLOBAL_BATCH_SIZE=$((MICRO_BATCH_SIZE*WORLD_SIZE*GRADIENT_ACCUMULATION_STEPS))
51
+
52
+ # Model parameters
53
+ source model_params.sh
54
+ MODEL_PARAM=("${PARAM_4516M[@]}")
55
+ NHIDDEN=${MODEL_PARAM[0]}
56
+ FFN_HIDDEN_SIZE=${MODEL_PARAM[1]}
57
+ KV_SIZE=${MODEL_PARAM[2]}
58
+ NHEADS=${MODEL_PARAM[3]}
59
+ NLAYERS=${MODEL_PARAM[4]}
60
+ SEQ_LEN=2048
61
+
62
+ echo "Model parameters: d_model $NHIDDEN ffw_size $FFN_HIDDEN_SIZE kv_size $KV_SIZE n_heads $NHEADS n_layers $NLAYERS"
63
+
64
+ SAVE_INTERVAL=10000
65
+
66
+ # Tokens: 84_000_000_000
67
+ # -> Samples: 41_015_625.0
68
+ TRAIN_SAMPLES=41_015_625
69
+
70
+ OPTIMIZER_ARGS=" \
71
+ --optimizer adam \
72
+ --adam-beta1 0.9 \
73
+ --adam-beta2 0.95 \
74
+ --adam-eps 1e-8 \
75
+ --lr 2e-4 \
76
+ --min-lr 2e-5 \
77
+ --lr-decay-style cosine \
78
+ --lr-decay-samples $TRAIN_SAMPLES \
79
+ --lr-warmup-samples 410_156 \
80
+ --clip-grad 1.0 \
81
+ --weight-decay 1e-1 \
82
+ "
83
+
84
+ GPT_ARGS=" \
85
+ --num-layers $NLAYERS \
86
+ --hidden-size $NHIDDEN \
87
+ --num-attention-heads $NHEADS \
88
+ --kv-channels $KV_SIZE \
89
+ --ffn-hidden-size $FFN_HIDDEN_SIZE \
90
+ --seq-length $SEQ_LEN \
91
+ --max-position-embeddings $SEQ_LEN \
92
+ --micro-batch-size $MICRO_BATCH_SIZE \
93
+ --global-batch-size $GLOBAL_BATCH_SIZE \
94
+ --train-samples $TRAIN_SAMPLES \
95
+ --vocab-file $VOCAB_FILE \
96
+ --merge-file $MERGE_FILE \
97
+ --clip-grad 1.0 \
98
+ --kill-switch-path $KILL_SWITCH_PATH \
99
+ --bf16 \
100
+ $OPTIMIZER_ARGS \
101
+ "
102
+
103
+ OUTPUT_ARGS=" \
104
+ --log-interval 10 \
105
+ --save-interval $SAVE_INTERVAL \
106
+ --eval-interval 1000 \
107
+ --eval-iters 100 \
108
+ --tensorboard-dir $TENSORBOARD_PATH \
109
+ --tensorboard-queue-size 5 \
110
+ --log-timers-to-tensorboard \
111
+ --log-batch-size-to-tensorboard \
112
+ --log-validation-ppl-to-tensorboard \
113
+ "
114
+
115
+ ZERO_STAGE=0
116
+
117
+ mkdir -p ds_configs
118
+ DS_CONFIG_PATH="ds_configs/$SLURM_JOB_ID.json"
119
+
120
+ cat <<EOF > $DS_CONFIG_PATH
121
+ {
122
+ "train_micro_batch_size_per_gpu": $MICRO_BATCH_SIZE,
123
+ "train_batch_size": $GLOBAL_BATCH_SIZE,
124
+ "gradient_clipping": 1.0,
125
+ "zero_optimization": {
126
+ "stage": $ZERO_STAGE
127
+ },
128
+ "bf16": {
129
+ "enabled": true
130
+ },
131
+ "steps_per_print": 2000,
132
+ "wall_clock_breakdown": false
133
+ }
134
+ EOF
135
+
136
+ DEEPSPEED_ARGS=" \
137
+ --deepspeed \
138
+ --deepspeed_config $DS_CONFIG_PATH \
139
+ --zero-stage $ZERO_STAGE \
140
+ "
141
+
142
+ CMD=" \
143
+ Megatron-DeepSpeed/pretrain_gpt.py \
144
+ --tensor-model-parallel-size $TP_SIZE \
145
+ --pipeline-model-parallel-size $PP_SIZE \
146
+ $GPT_ARGS \
147
+ $OUTPUT_ARGS \
148
+ --save $CHECKPOINT_PATH \
149
+ --load $CHECKPOINT_PATH \
150
+ --train-weighted-split-paths-path $TRAIN_DATA_PATH \
151
+ --valid-weighted-split-paths-path $VALID_DATA_PATH \
152
+ --data-impl mmap \
153
+ --seed 3 \
154
+ $DEEPSPEED_ARGS \
155
+ "
156
+
157
+ echo $CMD
158
+
159
+ echo "START $SLURM_JOBID: $(date)"
160
+
161
+ # bash launch_srun.sh $CMD
162
+ srun --label launch.sh $CMD
163
+
164
+ echo "END $SLURM_JOBID: $(date)"
sbatch_4b284b17bc4seed4.sh ADDED
@@ -0,0 +1,164 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #!/bin/bash
2
+ #SBATCH --exclude=nid007571,nid007112,nid006774,nid007502,nid007506,nid007507,nid005145,nid006692,nid007218,nid007123,nid006124,nid006123,nid007496,nid007237,nid006852,nid007206,nid006947,nid007212,nid006977,nid007222,nid005444,nid007219,nid007493,nid007221,nid005300,nid005619,nid006118,nid005203,nid006113,nid006481,nid007077,nid005208,nid005207,nid005879,nid005901
3
+ #SBATCH --nodes=32
4
+ #SBATCH --ntasks-per-node=1
5
+ #SBATCH --cpus-per-task=40
6
+ #SBATCH --mem=256G
7
+ #SBATCH -p standard-g
8
+ #SBATCH -t 48:00:00
9
+ #SBATCH --gpus-per-node=mi250:8
10
+ #SBATCH --exclusive=user
11
+ #SBATCH --hint=nomultithread
12
+ #SBATCH --account=project_462000119
13
+ #SBATCH -o logs/%j.out
14
+ #SBATCH -e logs/%j.err
15
+
16
+ VARIANT=4b284b17bc4seed4
17
+
18
+ # if run without sbatch, invoke here
19
+ if [ -z $SLURM_JOB_ID ]; then
20
+ mkdir -p logs
21
+ sbatch "$0"
22
+ exit
23
+ fi
24
+
25
+ set -euo pipefail
26
+
27
+ # symlink logs/latest.out and logs/latest.err
28
+ ln -f -s $SLURM_JOB_ID.out logs/latest.out
29
+ ln -f -s $SLURM_JOB_ID.err logs/latest.err
30
+
31
+ KILL_SWITCH_PATH=kill-switch-$VARIANT
32
+ CHECKPOINT_PATH=checkpoints_$VARIANT
33
+ TENSORBOARD_PATH=tensorboard_$VARIANT
34
+
35
+ # Data
36
+ VOCAB_FILE="gpt2/vocab.json"
37
+ MERGE_FILE="gpt2/merges.txt"
38
+
39
+ TRAIN_DATA_PATH=train17b.txt
40
+ # "train: 1.0 0:1 /scratch/project_462000119/data/c4_subsampled/gpt2tok_c4_en_17B_text_document"
41
+ VALID_DATA_PATH=val.txt
42
+ # "validation: 1.0 0:1 /scratch/project_462000119/data/c4_validation/gpt2tok_c4validation_rerun_text_document"
43
+
44
+ PP_SIZE=1
45
+ TP_SIZE=2
46
+
47
+ MICRO_BATCH_SIZE=2
48
+ GRADIENT_ACCUMULATION_STEPS=1
49
+ WORLD_SIZE=$((SLURM_GPUS_ON_NODE*SLURM_JOB_NUM_NODES))
50
+ GLOBAL_BATCH_SIZE=$((MICRO_BATCH_SIZE*WORLD_SIZE*GRADIENT_ACCUMULATION_STEPS))
51
+
52
+ # Model parameters
53
+ source model_params.sh
54
+ MODEL_PARAM=("${PARAM_4516M[@]}")
55
+ NHIDDEN=${MODEL_PARAM[0]}
56
+ FFN_HIDDEN_SIZE=${MODEL_PARAM[1]}
57
+ KV_SIZE=${MODEL_PARAM[2]}
58
+ NHEADS=${MODEL_PARAM[3]}
59
+ NLAYERS=${MODEL_PARAM[4]}
60
+ SEQ_LEN=2048
61
+
62
+ echo "Model parameters: d_model $NHIDDEN ffw_size $FFN_HIDDEN_SIZE kv_size $KV_SIZE n_heads $NHEADS n_layers $NLAYERS"
63
+
64
+ SAVE_INTERVAL=10000
65
+
66
+ # Tokens: 84_000_000_000
67
+ # -> Samples: 41_015_625.0
68
+ TRAIN_SAMPLES=41_015_625
69
+
70
+ OPTIMIZER_ARGS=" \
71
+ --optimizer adam \
72
+ --adam-beta1 0.9 \
73
+ --adam-beta2 0.95 \
74
+ --adam-eps 1e-8 \
75
+ --lr 2e-4 \
76
+ --min-lr 2e-5 \
77
+ --lr-decay-style cosine \
78
+ --lr-decay-samples $TRAIN_SAMPLES \
79
+ --lr-warmup-samples 410_156 \
80
+ --clip-grad 1.0 \
81
+ --weight-decay 1e-1 \
82
+ "
83
+
84
+ GPT_ARGS=" \
85
+ --num-layers $NLAYERS \
86
+ --hidden-size $NHIDDEN \
87
+ --num-attention-heads $NHEADS \
88
+ --kv-channels $KV_SIZE \
89
+ --ffn-hidden-size $FFN_HIDDEN_SIZE \
90
+ --seq-length $SEQ_LEN \
91
+ --max-position-embeddings $SEQ_LEN \
92
+ --micro-batch-size $MICRO_BATCH_SIZE \
93
+ --global-batch-size $GLOBAL_BATCH_SIZE \
94
+ --train-samples $TRAIN_SAMPLES \
95
+ --vocab-file $VOCAB_FILE \
96
+ --merge-file $MERGE_FILE \
97
+ --clip-grad 1.0 \
98
+ --kill-switch-path $KILL_SWITCH_PATH \
99
+ --bf16 \
100
+ $OPTIMIZER_ARGS \
101
+ "
102
+
103
+ OUTPUT_ARGS=" \
104
+ --log-interval 10 \
105
+ --save-interval $SAVE_INTERVAL \
106
+ --eval-interval 1000 \
107
+ --eval-iters 100 \
108
+ --tensorboard-dir $TENSORBOARD_PATH \
109
+ --tensorboard-queue-size 5 \
110
+ --log-timers-to-tensorboard \
111
+ --log-batch-size-to-tensorboard \
112
+ --log-validation-ppl-to-tensorboard \
113
+ "
114
+
115
+ ZERO_STAGE=0
116
+
117
+ mkdir -p ds_configs
118
+ DS_CONFIG_PATH="ds_configs/$SLURM_JOB_ID.json"
119
+
120
+ cat <<EOF > $DS_CONFIG_PATH
121
+ {
122
+ "train_micro_batch_size_per_gpu": $MICRO_BATCH_SIZE,
123
+ "train_batch_size": $GLOBAL_BATCH_SIZE,
124
+ "gradient_clipping": 1.0,
125
+ "zero_optimization": {
126
+ "stage": $ZERO_STAGE
127
+ },
128
+ "bf16": {
129
+ "enabled": true
130
+ },
131
+ "steps_per_print": 2000,
132
+ "wall_clock_breakdown": false
133
+ }
134
+ EOF
135
+
136
+ DEEPSPEED_ARGS=" \
137
+ --deepspeed \
138
+ --deepspeed_config $DS_CONFIG_PATH \
139
+ --zero-stage $ZERO_STAGE \
140
+ "
141
+
142
+ CMD=" \
143
+ Megatron-DeepSpeed/pretrain_gpt.py \
144
+ --tensor-model-parallel-size $TP_SIZE \
145
+ --pipeline-model-parallel-size $PP_SIZE \
146
+ $GPT_ARGS \
147
+ $OUTPUT_ARGS \
148
+ --save $CHECKPOINT_PATH \
149
+ --load $CHECKPOINT_PATH \
150
+ --train-weighted-split-paths-path $TRAIN_DATA_PATH \
151
+ --valid-weighted-split-paths-path $VALID_DATA_PATH \
152
+ --data-impl mmap \
153
+ --seed 4 \
154
+ $DEEPSPEED_ARGS \
155
+ "
156
+
157
+ echo $CMD
158
+
159
+ echo "START $SLURM_JOBID: $(date)"
160
+
161
+ # bash launch_srun.sh $CMD
162
+ srun --label launch.sh $CMD
163
+
164
+ echo "END $SLURM_JOBID: $(date)"
sbatch_4b284b21bc4seed1.sh ADDED
@@ -0,0 +1,164 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #!/bin/bash
2
+ #SBATCH --exclude=nid007571,nid007112,nid006774,nid007502,nid007506,nid007507,nid005145,nid006692,nid007218,nid007123,nid006124,nid006123,nid007496,nid007237,nid006852,nid007206,nid006947,nid007212,nid006977,nid007222,nid005444,nid007219,nid007493,nid007221,nid005300,nid005619,nid006118,nid005203,nid006113,nid006481,nid007077,nid005208,nid005207,nid005879,nid005901
3
+ #SBATCH --nodes=32
4
+ #SBATCH --ntasks-per-node=1
5
+ #SBATCH --cpus-per-task=40
6
+ #SBATCH --mem=256G
7
+ #SBATCH -p standard-g
8
+ #SBATCH -t 48:00:00
9
+ #SBATCH --gpus-per-node=mi250:8
10
+ #SBATCH --exclusive=user
11
+ #SBATCH --hint=nomultithread
12
+ #SBATCH --account=project_462000119
13
+ #SBATCH -o logs/%j.out
14
+ #SBATCH -e logs/%j.err
15
+
16
+ VARIANT=4b284b21bc4seed1
17
+
18
+ # if run without sbatch, invoke here
19
+ if [ -z $SLURM_JOB_ID ]; then
20
+ mkdir -p logs
21
+ sbatch "$0"
22
+ exit
23
+ fi
24
+
25
+ set -euo pipefail
26
+
27
+ # symlink logs/latest.out and logs/latest.err
28
+ ln -f -s $SLURM_JOB_ID.out logs/latest.out
29
+ ln -f -s $SLURM_JOB_ID.err logs/latest.err
30
+
31
+ KILL_SWITCH_PATH=kill-switch-$VARIANT
32
+ CHECKPOINT_PATH=checkpoints_$VARIANT
33
+ TENSORBOARD_PATH=tensorboard_$VARIANT
34
+
35
+ # Data
36
+ VOCAB_FILE="gpt2/vocab.json"
37
+ MERGE_FILE="gpt2/merges.txt"
38
+
39
+ TRAIN_DATA_PATH=train21b.txt
40
+ # "train: 1.0 0:1 /scratch/project_462000119/data/c4_subsampled/gpt2tok_c4_en_21B_text_document"
41
+ VALID_DATA_PATH=val.txt
42
+ # "validation: 1.0 0:1 /scratch/project_462000119/data/c4_validation/gpt2tok_c4validation_rerun_text_document"
43
+
44
+ PP_SIZE=1
45
+ TP_SIZE=2
46
+
47
+ MICRO_BATCH_SIZE=2
48
+ GRADIENT_ACCUMULATION_STEPS=1
49
+ WORLD_SIZE=$((SLURM_GPUS_ON_NODE*SLURM_JOB_NUM_NODES))
50
+ GLOBAL_BATCH_SIZE=$((MICRO_BATCH_SIZE*WORLD_SIZE*GRADIENT_ACCUMULATION_STEPS))
51
+
52
+ # Model parameters
53
+ source model_params.sh
54
+ MODEL_PARAM=("${PARAM_4516M[@]}")
55
+ NHIDDEN=${MODEL_PARAM[0]}
56
+ FFN_HIDDEN_SIZE=${MODEL_PARAM[1]}
57
+ KV_SIZE=${MODEL_PARAM[2]}
58
+ NHEADS=${MODEL_PARAM[3]}
59
+ NLAYERS=${MODEL_PARAM[4]}
60
+ SEQ_LEN=2048
61
+
62
+ echo "Model parameters: d_model $NHIDDEN ffw_size $FFN_HIDDEN_SIZE kv_size $KV_SIZE n_heads $NHEADS n_layers $NLAYERS"
63
+
64
+ SAVE_INTERVAL=10000
65
+
66
+ # Tokens: 84_000_000_000
67
+ # -> Samples: 41_015_625.0
68
+ TRAIN_SAMPLES=41_015_625
69
+
70
+ OPTIMIZER_ARGS=" \
71
+ --optimizer adam \
72
+ --adam-beta1 0.9 \
73
+ --adam-beta2 0.95 \
74
+ --adam-eps 1e-8 \
75
+ --lr 2e-4 \
76
+ --min-lr 2e-5 \
77
+ --lr-decay-style cosine \
78
+ --lr-decay-samples $TRAIN_SAMPLES \
79
+ --lr-warmup-samples 410_156 \
80
+ --clip-grad 1.0 \
81
+ --weight-decay 1e-1 \
82
+ "
83
+
84
+ GPT_ARGS=" \
85
+ --num-layers $NLAYERS \
86
+ --hidden-size $NHIDDEN \
87
+ --num-attention-heads $NHEADS \
88
+ --kv-channels $KV_SIZE \
89
+ --ffn-hidden-size $FFN_HIDDEN_SIZE \
90
+ --seq-length $SEQ_LEN \
91
+ --max-position-embeddings $SEQ_LEN \
92
+ --micro-batch-size $MICRO_BATCH_SIZE \
93
+ --global-batch-size $GLOBAL_BATCH_SIZE \
94
+ --train-samples $TRAIN_SAMPLES \
95
+ --vocab-file $VOCAB_FILE \
96
+ --merge-file $MERGE_FILE \
97
+ --clip-grad 1.0 \
98
+ --kill-switch-path $KILL_SWITCH_PATH \
99
+ --bf16 \
100
+ $OPTIMIZER_ARGS \
101
+ "
102
+
103
+ OUTPUT_ARGS=" \
104
+ --log-interval 10 \
105
+ --save-interval $SAVE_INTERVAL \
106
+ --eval-interval 1000 \
107
+ --eval-iters 100 \
108
+ --tensorboard-dir $TENSORBOARD_PATH \
109
+ --tensorboard-queue-size 5 \
110
+ --log-timers-to-tensorboard \
111
+ --log-batch-size-to-tensorboard \
112
+ --log-validation-ppl-to-tensorboard \
113
+ "
114
+
115
+ ZERO_STAGE=0
116
+
117
+ mkdir -p ds_configs
118
+ DS_CONFIG_PATH="ds_configs/$SLURM_JOB_ID.json"
119
+
120
+ cat <<EOF > $DS_CONFIG_PATH
121
+ {
122
+ "train_micro_batch_size_per_gpu": $MICRO_BATCH_SIZE,
123
+ "train_batch_size": $GLOBAL_BATCH_SIZE,
124
+ "gradient_clipping": 1.0,
125
+ "zero_optimization": {
126
+ "stage": $ZERO_STAGE
127
+ },
128
+ "bf16": {
129
+ "enabled": true
130
+ },
131
+ "steps_per_print": 2000,
132
+ "wall_clock_breakdown": false
133
+ }
134
+ EOF
135
+
136
+ DEEPSPEED_ARGS=" \
137
+ --deepspeed \
138
+ --deepspeed_config $DS_CONFIG_PATH \
139
+ --zero-stage $ZERO_STAGE \
140
+ "
141
+
142
+ CMD=" \
143
+ Megatron-DeepSpeed/pretrain_gpt.py \
144
+ --tensor-model-parallel-size $TP_SIZE \
145
+ --pipeline-model-parallel-size $PP_SIZE \
146
+ $GPT_ARGS \
147
+ $OUTPUT_ARGS \
148
+ --save $CHECKPOINT_PATH \
149
+ --load $CHECKPOINT_PATH \
150
+ --train-weighted-split-paths-path $TRAIN_DATA_PATH \
151
+ --valid-weighted-split-paths-path $VALID_DATA_PATH \
152
+ --data-impl mmap \
153
+ --seed 1 \
154
+ $DEEPSPEED_ARGS \
155
+ "
156
+
157
+ echo $CMD
158
+
159
+ echo "START $SLURM_JOBID: $(date)"
160
+
161
+ # bash launch_srun.sh $CMD
162
+ srun --label launch.sh $CMD
163
+
164
+ echo "END $SLURM_JOBID: $(date)"
sbatch_4b284b21bc4seed2.sh ADDED
@@ -0,0 +1,164 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #!/bin/bash
2
+ #SBATCH --exclude=nid007571,nid007112,nid006774,nid007502,nid007506,nid007507,nid005145,nid006692,nid007218,nid007123,nid006124,nid006123,nid007496,nid007237,nid006852,nid007206,nid006947,nid007212,nid006977,nid007222,nid005444,nid007219,nid007493,nid007221,nid005300,nid005619,nid006118,nid005203,nid006113,nid006481,nid007077,nid005208,nid005207,nid005879,nid005901
3
+ #SBATCH --nodes=32
4
+ #SBATCH --ntasks-per-node=1
5
+ #SBATCH --cpus-per-task=40
6
+ #SBATCH --mem=256G
7
+ #SBATCH -p standard-g
8
+ #SBATCH -t 48:00:00
9
+ #SBATCH --gpus-per-node=mi250:8
10
+ #SBATCH --exclusive=user
11
+ #SBATCH --hint=nomultithread
12
+ #SBATCH --account=project_462000119
13
+ #SBATCH -o logs/%j.out
14
+ #SBATCH -e logs/%j.err
15
+
16
+ VARIANT=4b284b21bc4seed2
17
+
18
+ # if run without sbatch, invoke here
19
+ if [ -z $SLURM_JOB_ID ]; then
20
+ mkdir -p logs
21
+ sbatch "$0"
22
+ exit
23
+ fi
24
+
25
+ set -euo pipefail
26
+
27
+ # symlink logs/latest.out and logs/latest.err
28
+ ln -f -s $SLURM_JOB_ID.out logs/latest.out
29
+ ln -f -s $SLURM_JOB_ID.err logs/latest.err
30
+
31
+ KILL_SWITCH_PATH=kill-switch-$VARIANT
32
+ CHECKPOINT_PATH=checkpoints_$VARIANT
33
+ TENSORBOARD_PATH=tensorboard_$VARIANT
34
+
35
+ # Data
36
+ VOCAB_FILE="gpt2/vocab.json"
37
+ MERGE_FILE="gpt2/merges.txt"
38
+
39
+ TRAIN_DATA_PATH=train21b.txt
40
+ # "train: 1.0 0:1 /scratch/project_462000119/data/c4_subsampled/gpt2tok_c4_en_21B_text_document"
41
+ VALID_DATA_PATH=val.txt
42
+ # "validation: 1.0 0:1 /scratch/project_462000119/data/c4_validation/gpt2tok_c4validation_rerun_text_document"
43
+
44
+ PP_SIZE=1
45
+ TP_SIZE=2
46
+
47
+ MICRO_BATCH_SIZE=2
48
+ GRADIENT_ACCUMULATION_STEPS=1
49
+ WORLD_SIZE=$((SLURM_GPUS_ON_NODE*SLURM_JOB_NUM_NODES))
50
+ GLOBAL_BATCH_SIZE=$((MICRO_BATCH_SIZE*WORLD_SIZE*GRADIENT_ACCUMULATION_STEPS))
51
+
52
+ # Model parameters
53
+ source model_params.sh
54
+ MODEL_PARAM=("${PARAM_4516M[@]}")
55
+ NHIDDEN=${MODEL_PARAM[0]}
56
+ FFN_HIDDEN_SIZE=${MODEL_PARAM[1]}
57
+ KV_SIZE=${MODEL_PARAM[2]}
58
+ NHEADS=${MODEL_PARAM[3]}
59
+ NLAYERS=${MODEL_PARAM[4]}
60
+ SEQ_LEN=2048
61
+
62
+ echo "Model parameters: d_model $NHIDDEN ffw_size $FFN_HIDDEN_SIZE kv_size $KV_SIZE n_heads $NHEADS n_layers $NLAYERS"
63
+
64
+ SAVE_INTERVAL=10000
65
+
66
+ # Tokens: 84_000_000_000
67
+ # -> Samples: 41_015_625.0
68
+ TRAIN_SAMPLES=41_015_625
69
+
70
+ OPTIMIZER_ARGS=" \
71
+ --optimizer adam \
72
+ --adam-beta1 0.9 \
73
+ --adam-beta2 0.95 \
74
+ --adam-eps 1e-8 \
75
+ --lr 2e-4 \
76
+ --min-lr 2e-5 \
77
+ --lr-decay-style cosine \
78
+ --lr-decay-samples $TRAIN_SAMPLES \
79
+ --lr-warmup-samples 410_156 \
80
+ --clip-grad 1.0 \
81
+ --weight-decay 1e-1 \
82
+ "
83
+
84
+ GPT_ARGS=" \
85
+ --num-layers $NLAYERS \
86
+ --hidden-size $NHIDDEN \
87
+ --num-attention-heads $NHEADS \
88
+ --kv-channels $KV_SIZE \
89
+ --ffn-hidden-size $FFN_HIDDEN_SIZE \
90
+ --seq-length $SEQ_LEN \
91
+ --max-position-embeddings $SEQ_LEN \
92
+ --micro-batch-size $MICRO_BATCH_SIZE \
93
+ --global-batch-size $GLOBAL_BATCH_SIZE \
94
+ --train-samples $TRAIN_SAMPLES \
95
+ --vocab-file $VOCAB_FILE \
96
+ --merge-file $MERGE_FILE \
97
+ --clip-grad 1.0 \
98
+ --kill-switch-path $KILL_SWITCH_PATH \
99
+ --bf16 \
100
+ $OPTIMIZER_ARGS \
101
+ "
102
+
103
+ OUTPUT_ARGS=" \
104
+ --log-interval 10 \
105
+ --save-interval $SAVE_INTERVAL \
106
+ --eval-interval 1000 \
107
+ --eval-iters 100 \
108
+ --tensorboard-dir $TENSORBOARD_PATH \
109
+ --tensorboard-queue-size 5 \
110
+ --log-timers-to-tensorboard \
111
+ --log-batch-size-to-tensorboard \
112
+ --log-validation-ppl-to-tensorboard \
113
+ "
114
+
115
+ ZERO_STAGE=0
116
+
117
+ mkdir -p ds_configs
118
+ DS_CONFIG_PATH="ds_configs/$SLURM_JOB_ID.json"
119
+
120
+ cat <<EOF > $DS_CONFIG_PATH
121
+ {
122
+ "train_micro_batch_size_per_gpu": $MICRO_BATCH_SIZE,
123
+ "train_batch_size": $GLOBAL_BATCH_SIZE,
124
+ "gradient_clipping": 1.0,
125
+ "zero_optimization": {
126
+ "stage": $ZERO_STAGE
127
+ },
128
+ "bf16": {
129
+ "enabled": true
130
+ },
131
+ "steps_per_print": 2000,
132
+ "wall_clock_breakdown": false
133
+ }
134
+ EOF
135
+
136
+ DEEPSPEED_ARGS=" \
137
+ --deepspeed \
138
+ --deepspeed_config $DS_CONFIG_PATH \
139
+ --zero-stage $ZERO_STAGE \
140
+ "
141
+
142
+ CMD=" \
143
+ Megatron-DeepSpeed/pretrain_gpt.py \
144
+ --tensor-model-parallel-size $TP_SIZE \
145
+ --pipeline-model-parallel-size $PP_SIZE \
146
+ $GPT_ARGS \
147
+ $OUTPUT_ARGS \
148
+ --save $CHECKPOINT_PATH \
149
+ --load $CHECKPOINT_PATH \
150
+ --train-weighted-split-paths-path $TRAIN_DATA_PATH \
151
+ --valid-weighted-split-paths-path $VALID_DATA_PATH \
152
+ --data-impl mmap \
153
+ --seed 2 \
154
+ $DEEPSPEED_ARGS \
155
+ "
156
+
157
+ echo $CMD
158
+
159
+ echo "START $SLURM_JOBID: $(date)"
160
+
161
+ # bash launch_srun.sh $CMD
162
+ srun --label launch.sh $CMD
163
+
164
+ echo "END $SLURM_JOBID: $(date)"
sbatch_4b284b21bc4seed3.sh ADDED
@@ -0,0 +1,164 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #!/bin/bash
2
+ #SBATCH --exclude=nid007571,nid007112,nid006774,nid007502,nid007506,nid007507,nid005145,nid006692,nid007218,nid007123,nid006124,nid006123,nid007496,nid007237,nid006852,nid007206,nid006947,nid007212,nid006977,nid007222,nid005444,nid007219,nid007493,nid007221,nid005300,nid005619,nid006118,nid005203,nid006113,nid006481,nid007077,nid005208,nid005207,nid005879,nid005901
3
+ #SBATCH --nodes=32
4
+ #SBATCH --ntasks-per-node=1
5
+ #SBATCH --cpus-per-task=40
6
+ #SBATCH --mem=256G
7
+ #SBATCH -p standard-g
8
+ #SBATCH -t 48:00:00
9
+ #SBATCH --gpus-per-node=mi250:8
10
+ #SBATCH --exclusive=user
11
+ #SBATCH --hint=nomultithread
12
+ #SBATCH --account=project_462000119
13
+ #SBATCH -o logs/%j.out
14
+ #SBATCH -e logs/%j.err
15
+
16
+ VARIANT=4b284b21bc4seed3
17
+
18
+ # if run without sbatch, invoke here
19
+ if [ -z $SLURM_JOB_ID ]; then
20
+ mkdir -p logs
21
+ sbatch "$0"
22
+ exit
23
+ fi
24
+
25
+ set -euo pipefail
26
+
27
+ # symlink logs/latest.out and logs/latest.err
28
+ ln -f -s $SLURM_JOB_ID.out logs/latest.out
29
+ ln -f -s $SLURM_JOB_ID.err logs/latest.err
30
+
31
+ KILL_SWITCH_PATH=kill-switch-$VARIANT
32
+ CHECKPOINT_PATH=checkpoints_$VARIANT
33
+ TENSORBOARD_PATH=tensorboard_$VARIANT
34
+
35
+ # Data
36
+ VOCAB_FILE="gpt2/vocab.json"
37
+ MERGE_FILE="gpt2/merges.txt"
38
+
39
+ TRAIN_DATA_PATH=train21b.txt
40
+ # "train: 1.0 0:1 /scratch/project_462000119/data/c4_subsampled/gpt2tok_c4_en_21B_text_document"
41
+ VALID_DATA_PATH=val.txt
42
+ # "validation: 1.0 0:1 /scratch/project_462000119/data/c4_validation/gpt2tok_c4validation_rerun_text_document"
43
+
44
+ PP_SIZE=1
45
+ TP_SIZE=2
46
+
47
+ MICRO_BATCH_SIZE=2
48
+ GRADIENT_ACCUMULATION_STEPS=1
49
+ WORLD_SIZE=$((SLURM_GPUS_ON_NODE*SLURM_JOB_NUM_NODES))
50
+ GLOBAL_BATCH_SIZE=$((MICRO_BATCH_SIZE*WORLD_SIZE*GRADIENT_ACCUMULATION_STEPS))
51
+
52
+ # Model parameters
53
+ source model_params.sh
54
+ MODEL_PARAM=("${PARAM_4516M[@]}")
55
+ NHIDDEN=${MODEL_PARAM[0]}
56
+ FFN_HIDDEN_SIZE=${MODEL_PARAM[1]}
57
+ KV_SIZE=${MODEL_PARAM[2]}
58
+ NHEADS=${MODEL_PARAM[3]}
59
+ NLAYERS=${MODEL_PARAM[4]}
60
+ SEQ_LEN=2048
61
+
62
+ echo "Model parameters: d_model $NHIDDEN ffw_size $FFN_HIDDEN_SIZE kv_size $KV_SIZE n_heads $NHEADS n_layers $NLAYERS"
63
+
64
+ SAVE_INTERVAL=10000
65
+
66
+ # Tokens: 84_000_000_000
67
+ # -> Samples: 41_015_625.0
68
+ TRAIN_SAMPLES=41_015_625
69
+
70
+ OPTIMIZER_ARGS=" \
71
+ --optimizer adam \
72
+ --adam-beta1 0.9 \
73
+ --adam-beta2 0.95 \
74
+ --adam-eps 1e-8 \
75
+ --lr 2e-4 \
76
+ --min-lr 2e-5 \
77
+ --lr-decay-style cosine \
78
+ --lr-decay-samples $TRAIN_SAMPLES \
79
+ --lr-warmup-samples 410_156 \
80
+ --clip-grad 1.0 \
81
+ --weight-decay 1e-1 \
82
+ "
83
+
84
+ GPT_ARGS=" \
85
+ --num-layers $NLAYERS \
86
+ --hidden-size $NHIDDEN \
87
+ --num-attention-heads $NHEADS \
88
+ --kv-channels $KV_SIZE \
89
+ --ffn-hidden-size $FFN_HIDDEN_SIZE \
90
+ --seq-length $SEQ_LEN \
91
+ --max-position-embeddings $SEQ_LEN \
92
+ --micro-batch-size $MICRO_BATCH_SIZE \
93
+ --global-batch-size $GLOBAL_BATCH_SIZE \
94
+ --train-samples $TRAIN_SAMPLES \
95
+ --vocab-file $VOCAB_FILE \
96
+ --merge-file $MERGE_FILE \
97
+ --clip-grad 1.0 \
98
+ --kill-switch-path $KILL_SWITCH_PATH \
99
+ --bf16 \
100
+ $OPTIMIZER_ARGS \
101
+ "
102
+
103
+ OUTPUT_ARGS=" \
104
+ --log-interval 10 \
105
+ --save-interval $SAVE_INTERVAL \
106
+ --eval-interval 1000 \
107
+ --eval-iters 100 \
108
+ --tensorboard-dir $TENSORBOARD_PATH \
109
+ --tensorboard-queue-size 5 \
110
+ --log-timers-to-tensorboard \
111
+ --log-batch-size-to-tensorboard \
112
+ --log-validation-ppl-to-tensorboard \
113
+ "
114
+
115
+ ZERO_STAGE=0
116
+
117
+ mkdir -p ds_configs
118
+ DS_CONFIG_PATH="ds_configs/$SLURM_JOB_ID.json"
119
+
120
+ cat <<EOF > $DS_CONFIG_PATH
121
+ {
122
+ "train_micro_batch_size_per_gpu": $MICRO_BATCH_SIZE,
123
+ "train_batch_size": $GLOBAL_BATCH_SIZE,
124
+ "gradient_clipping": 1.0,
125
+ "zero_optimization": {
126
+ "stage": $ZERO_STAGE
127
+ },
128
+ "bf16": {
129
+ "enabled": true
130
+ },
131
+ "steps_per_print": 2000,
132
+ "wall_clock_breakdown": false
133
+ }
134
+ EOF
135
+
136
+ DEEPSPEED_ARGS=" \
137
+ --deepspeed \
138
+ --deepspeed_config $DS_CONFIG_PATH \
139
+ --zero-stage $ZERO_STAGE \
140
+ "
141
+
142
+ CMD=" \
143
+ Megatron-DeepSpeed/pretrain_gpt.py \
144
+ --tensor-model-parallel-size $TP_SIZE \
145
+ --pipeline-model-parallel-size $PP_SIZE \
146
+ $GPT_ARGS \
147
+ $OUTPUT_ARGS \
148
+ --save $CHECKPOINT_PATH \
149
+ --load $CHECKPOINT_PATH \
150
+ --train-weighted-split-paths-path $TRAIN_DATA_PATH \
151
+ --valid-weighted-split-paths-path $VALID_DATA_PATH \
152
+ --data-impl mmap \
153
+ --seed 3 \
154
+ $DEEPSPEED_ARGS \
155
+ "
156
+
157
+ echo $CMD
158
+
159
+ echo "START $SLURM_JOBID: $(date)"
160
+
161
+ # bash launch_srun.sh $CMD
162
+ srun --label launch.sh $CMD
163
+
164
+ echo "END $SLURM_JOBID: $(date)"
sbatch_4b284b21bc4seed4.sh ADDED
@@ -0,0 +1,164 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #!/bin/bash
2
+ #SBATCH --exclude=nid007571,nid007112,nid006774,nid007502,nid007506,nid007507,nid005145,nid006692,nid007218,nid007123,nid006124,nid006123,nid007496,nid007237,nid006852,nid007206,nid006947,nid007212,nid006977,nid007222,nid005444,nid007219,nid007493,nid007221,nid005300,nid005619,nid006118,nid005203,nid006113,nid006481,nid007077,nid005208,nid005207,nid005879,nid005901
3
+ #SBATCH --nodes=32
4
+ #SBATCH --ntasks-per-node=1
5
+ #SBATCH --cpus-per-task=40
6
+ #SBATCH --mem=256G
7
+ #SBATCH -p standard-g
8
+ #SBATCH -t 48:00:00
9
+ #SBATCH --gpus-per-node=mi250:8
10
+ #SBATCH --exclusive=user
11
+ #SBATCH --hint=nomultithread
12
+ #SBATCH --account=project_462000119
13
+ #SBATCH -o logs/%j.out
14
+ #SBATCH -e logs/%j.err
15
+
16
+ VARIANT=4b284b21bc4seed4
17
+
18
+ # if run without sbatch, invoke here
19
+ if [ -z $SLURM_JOB_ID ]; then
20
+ mkdir -p logs
21
+ sbatch "$0"
22
+ exit
23
+ fi
24
+
25
+ set -euo pipefail
26
+
27
+ # symlink logs/latest.out and logs/latest.err
28
+ ln -f -s $SLURM_JOB_ID.out logs/latest.out
29
+ ln -f -s $SLURM_JOB_ID.err logs/latest.err
30
+
31
+ KILL_SWITCH_PATH=kill-switch-$VARIANT
32
+ CHECKPOINT_PATH=checkpoints_$VARIANT
33
+ TENSORBOARD_PATH=tensorboard_$VARIANT
34
+
35
+ # Data
36
+ VOCAB_FILE="gpt2/vocab.json"
37
+ MERGE_FILE="gpt2/merges.txt"
38
+
39
+ TRAIN_DATA_PATH=train21b.txt
40
+ # "train: 1.0 0:1 /scratch/project_462000119/data/c4_subsampled/gpt2tok_c4_en_21B_text_document"
41
+ VALID_DATA_PATH=val.txt
42
+ # "validation: 1.0 0:1 /scratch/project_462000119/data/c4_validation/gpt2tok_c4validation_rerun_text_document"
43
+
44
+ PP_SIZE=1
45
+ TP_SIZE=2
46
+
47
+ MICRO_BATCH_SIZE=2
48
+ GRADIENT_ACCUMULATION_STEPS=1
49
+ WORLD_SIZE=$((SLURM_GPUS_ON_NODE*SLURM_JOB_NUM_NODES))
50
+ GLOBAL_BATCH_SIZE=$((MICRO_BATCH_SIZE*WORLD_SIZE*GRADIENT_ACCUMULATION_STEPS))
51
+
52
+ # Model parameters
53
+ source model_params.sh
54
+ MODEL_PARAM=("${PARAM_4516M[@]}")
55
+ NHIDDEN=${MODEL_PARAM[0]}
56
+ FFN_HIDDEN_SIZE=${MODEL_PARAM[1]}
57
+ KV_SIZE=${MODEL_PARAM[2]}
58
+ NHEADS=${MODEL_PARAM[3]}
59
+ NLAYERS=${MODEL_PARAM[4]}
60
+ SEQ_LEN=2048
61
+
62
+ echo "Model parameters: d_model $NHIDDEN ffw_size $FFN_HIDDEN_SIZE kv_size $KV_SIZE n_heads $NHEADS n_layers $NLAYERS"
63
+
64
+ SAVE_INTERVAL=10000
65
+
66
+ # Tokens: 84_000_000_000
67
+ # -> Samples: 41_015_625.0
68
+ TRAIN_SAMPLES=41_015_625
69
+
70
+ OPTIMIZER_ARGS=" \
71
+ --optimizer adam \
72
+ --adam-beta1 0.9 \
73
+ --adam-beta2 0.95 \
74
+ --adam-eps 1e-8 \
75
+ --lr 2e-4 \
76
+ --min-lr 2e-5 \
77
+ --lr-decay-style cosine \
78
+ --lr-decay-samples $TRAIN_SAMPLES \
79
+ --lr-warmup-samples 410_156 \
80
+ --clip-grad 1.0 \
81
+ --weight-decay 1e-1 \
82
+ "
83
+
84
+ GPT_ARGS=" \
85
+ --num-layers $NLAYERS \
86
+ --hidden-size $NHIDDEN \
87
+ --num-attention-heads $NHEADS \
88
+ --kv-channels $KV_SIZE \
89
+ --ffn-hidden-size $FFN_HIDDEN_SIZE \
90
+ --seq-length $SEQ_LEN \
91
+ --max-position-embeddings $SEQ_LEN \
92
+ --micro-batch-size $MICRO_BATCH_SIZE \
93
+ --global-batch-size $GLOBAL_BATCH_SIZE \
94
+ --train-samples $TRAIN_SAMPLES \
95
+ --vocab-file $VOCAB_FILE \
96
+ --merge-file $MERGE_FILE \
97
+ --clip-grad 1.0 \
98
+ --kill-switch-path $KILL_SWITCH_PATH \
99
+ --bf16 \
100
+ $OPTIMIZER_ARGS \
101
+ "
102
+
103
+ OUTPUT_ARGS=" \
104
+ --log-interval 10 \
105
+ --save-interval $SAVE_INTERVAL \
106
+ --eval-interval 1000 \
107
+ --eval-iters 100 \
108
+ --tensorboard-dir $TENSORBOARD_PATH \
109
+ --tensorboard-queue-size 5 \
110
+ --log-timers-to-tensorboard \
111
+ --log-batch-size-to-tensorboard \
112
+ --log-validation-ppl-to-tensorboard \
113
+ "
114
+
115
+ ZERO_STAGE=0
116
+
117
+ mkdir -p ds_configs
118
+ DS_CONFIG_PATH="ds_configs/$SLURM_JOB_ID.json"
119
+
120
+ cat <<EOF > $DS_CONFIG_PATH
121
+ {
122
+ "train_micro_batch_size_per_gpu": $MICRO_BATCH_SIZE,
123
+ "train_batch_size": $GLOBAL_BATCH_SIZE,
124
+ "gradient_clipping": 1.0,
125
+ "zero_optimization": {
126
+ "stage": $ZERO_STAGE
127
+ },
128
+ "bf16": {
129
+ "enabled": true
130
+ },
131
+ "steps_per_print": 2000,
132
+ "wall_clock_breakdown": false
133
+ }
134
+ EOF
135
+
136
+ DEEPSPEED_ARGS=" \
137
+ --deepspeed \
138
+ --deepspeed_config $DS_CONFIG_PATH \
139
+ --zero-stage $ZERO_STAGE \
140
+ "
141
+
142
+ CMD=" \
143
+ Megatron-DeepSpeed/pretrain_gpt.py \
144
+ --tensor-model-parallel-size $TP_SIZE \
145
+ --pipeline-model-parallel-size $PP_SIZE \
146
+ $GPT_ARGS \
147
+ $OUTPUT_ARGS \
148
+ --save $CHECKPOINT_PATH \
149
+ --load $CHECKPOINT_PATH \
150
+ --train-weighted-split-paths-path $TRAIN_DATA_PATH \
151
+ --valid-weighted-split-paths-path $VALID_DATA_PATH \
152
+ --data-impl mmap \
153
+ --seed 4 \
154
+ $DEEPSPEED_ARGS \
155
+ "
156
+
157
+ echo $CMD
158
+
159
+ echo "START $SLURM_JOBID: $(date)"
160
+
161
+ # bash launch_srun.sh $CMD
162
+ srun --label launch.sh $CMD
163
+
164
+ echo "END $SLURM_JOBID: $(date)"
sbatch_4b284b28bc4seed1.sh ADDED
@@ -0,0 +1,164 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #!/bin/bash
2
+ #SBATCH --exclude=nid007571,nid007112,nid006774,nid007502,nid007506,nid007507,nid005145,nid006692,nid007218,nid007123,nid006124,nid006123,nid007496,nid007237,nid006852,nid007206,nid006947,nid007212,nid006977,nid007222,nid005444,nid007219,nid007493,nid007221,nid005300,nid005619,nid006118,nid005203,nid006113,nid006481,nid007077,nid005208,nid005207,nid005879,nid005901
3
+ #SBATCH --nodes=32
4
+ #SBATCH --ntasks-per-node=1
5
+ #SBATCH --cpus-per-task=40
6
+ #SBATCH --mem=256G
7
+ #SBATCH -p standard-g
8
+ #SBATCH -t 48:00:00
9
+ #SBATCH --gpus-per-node=mi250:8
10
+ #SBATCH --exclusive=user
11
+ #SBATCH --hint=nomultithread
12
+ #SBATCH --account=project_462000119
13
+ #SBATCH -o logs/%j.out
14
+ #SBATCH -e logs/%j.err
15
+
16
+ VARIANT=4b284b28bc4seed1
17
+
18
+ # if run without sbatch, invoke here
19
+ if [ -z $SLURM_JOB_ID ]; then
20
+ mkdir -p logs
21
+ sbatch "$0"
22
+ exit
23
+ fi
24
+
25
+ set -euo pipefail
26
+
27
+ # symlink logs/latest.out and logs/latest.err
28
+ ln -f -s $SLURM_JOB_ID.out logs/latest.out
29
+ ln -f -s $SLURM_JOB_ID.err logs/latest.err
30
+
31
+ KILL_SWITCH_PATH=kill-switch-$VARIANT
32
+ CHECKPOINT_PATH=checkpoints_$VARIANT
33
+ TENSORBOARD_PATH=tensorboard_$VARIANT
34
+
35
+ # Data
36
+ VOCAB_FILE="gpt2/vocab.json"
37
+ MERGE_FILE="gpt2/merges.txt"
38
+
39
+ TRAIN_DATA_PATH=train28b.txt
40
+ # "train: 1.0 0:1 /scratch/project_462000119/data/c4_subsampled/gpt2tok_c4_en_28B_text_document"
41
+ VALID_DATA_PATH=val.txt
42
+ # "validation: 1.0 0:1 /scratch/project_462000119/data/c4_validation/gpt2tok_c4validation_rerun_text_document"
43
+
44
+ PP_SIZE=1
45
+ TP_SIZE=2
46
+
47
+ MICRO_BATCH_SIZE=2
48
+ GRADIENT_ACCUMULATION_STEPS=1
49
+ WORLD_SIZE=$((SLURM_GPUS_ON_NODE*SLURM_JOB_NUM_NODES))
50
+ GLOBAL_BATCH_SIZE=$((MICRO_BATCH_SIZE*WORLD_SIZE*GRADIENT_ACCUMULATION_STEPS))
51
+
52
+ # Model parameters
53
+ source model_params.sh
54
+ MODEL_PARAM=("${PARAM_4516M[@]}")
55
+ NHIDDEN=${MODEL_PARAM[0]}
56
+ FFN_HIDDEN_SIZE=${MODEL_PARAM[1]}
57
+ KV_SIZE=${MODEL_PARAM[2]}
58
+ NHEADS=${MODEL_PARAM[3]}
59
+ NLAYERS=${MODEL_PARAM[4]}
60
+ SEQ_LEN=2048
61
+
62
+ echo "Model parameters: d_model $NHIDDEN ffw_size $FFN_HIDDEN_SIZE kv_size $KV_SIZE n_heads $NHEADS n_layers $NLAYERS"
63
+
64
+ SAVE_INTERVAL=10000
65
+
66
+ # Tokens: 84_000_000_000
67
+ # -> Samples: 41_015_625.0
68
+ TRAIN_SAMPLES=41_015_625
69
+
70
+ OPTIMIZER_ARGS=" \
71
+ --optimizer adam \
72
+ --adam-beta1 0.9 \
73
+ --adam-beta2 0.95 \
74
+ --adam-eps 1e-8 \
75
+ --lr 2e-4 \
76
+ --min-lr 2e-5 \
77
+ --lr-decay-style cosine \
78
+ --lr-decay-samples $TRAIN_SAMPLES \
79
+ --lr-warmup-samples 410_156 \
80
+ --clip-grad 1.0 \
81
+ --weight-decay 1e-1 \
82
+ "
83
+
84
+ GPT_ARGS=" \
85
+ --num-layers $NLAYERS \
86
+ --hidden-size $NHIDDEN \
87
+ --num-attention-heads $NHEADS \
88
+ --kv-channels $KV_SIZE \
89
+ --ffn-hidden-size $FFN_HIDDEN_SIZE \
90
+ --seq-length $SEQ_LEN \
91
+ --max-position-embeddings $SEQ_LEN \
92
+ --micro-batch-size $MICRO_BATCH_SIZE \
93
+ --global-batch-size $GLOBAL_BATCH_SIZE \
94
+ --train-samples $TRAIN_SAMPLES \
95
+ --vocab-file $VOCAB_FILE \
96
+ --merge-file $MERGE_FILE \
97
+ --clip-grad 1.0 \
98
+ --kill-switch-path $KILL_SWITCH_PATH \
99
+ --bf16 \
100
+ $OPTIMIZER_ARGS \
101
+ "
102
+
103
+ OUTPUT_ARGS=" \
104
+ --log-interval 10 \
105
+ --save-interval $SAVE_INTERVAL \
106
+ --eval-interval 1000 \
107
+ --eval-iters 100 \
108
+ --tensorboard-dir $TENSORBOARD_PATH \
109
+ --tensorboard-queue-size 5 \
110
+ --log-timers-to-tensorboard \
111
+ --log-batch-size-to-tensorboard \
112
+ --log-validation-ppl-to-tensorboard \
113
+ "
114
+
115
+ ZERO_STAGE=0
116
+
117
+ mkdir -p ds_configs
118
+ DS_CONFIG_PATH="ds_configs/$SLURM_JOB_ID.json"
119
+
120
+ cat <<EOF > $DS_CONFIG_PATH
121
+ {
122
+ "train_micro_batch_size_per_gpu": $MICRO_BATCH_SIZE,
123
+ "train_batch_size": $GLOBAL_BATCH_SIZE,
124
+ "gradient_clipping": 1.0,
125
+ "zero_optimization": {
126
+ "stage": $ZERO_STAGE
127
+ },
128
+ "bf16": {
129
+ "enabled": true
130
+ },
131
+ "steps_per_print": 2000,
132
+ "wall_clock_breakdown": false
133
+ }
134
+ EOF
135
+
136
+ DEEPSPEED_ARGS=" \
137
+ --deepspeed \
138
+ --deepspeed_config $DS_CONFIG_PATH \
139
+ --zero-stage $ZERO_STAGE \
140
+ "
141
+
142
+ CMD=" \
143
+ Megatron-DeepSpeed/pretrain_gpt.py \
144
+ --tensor-model-parallel-size $TP_SIZE \
145
+ --pipeline-model-parallel-size $PP_SIZE \
146
+ $GPT_ARGS \
147
+ $OUTPUT_ARGS \
148
+ --save $CHECKPOINT_PATH \
149
+ --load $CHECKPOINT_PATH \
150
+ --train-weighted-split-paths-path $TRAIN_DATA_PATH \
151
+ --valid-weighted-split-paths-path $VALID_DATA_PATH \
152
+ --data-impl mmap \
153
+ --seed 1 \
154
+ $DEEPSPEED_ARGS \
155
+ "
156
+
157
+ echo $CMD
158
+
159
+ echo "START $SLURM_JOBID: $(date)"
160
+
161
+ # bash launch_srun.sh $CMD
162
+ srun --label launch.sh $CMD
163
+
164
+ echo "END $SLURM_JOBID: $(date)"
sbatch_4b284b28bc4seed2.sh ADDED
@@ -0,0 +1,164 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #!/bin/bash
2
+ #SBATCH --exclude=nid007571,nid007112,nid006774,nid007502,nid007506,nid007507,nid005145,nid006692,nid007218,nid007123,nid006124,nid006123,nid007496,nid007237,nid006852,nid007206,nid006947,nid007212,nid006977,nid007222,nid005444,nid007219,nid007493,nid007221,nid005300,nid005619,nid006118,nid005203,nid006113,nid006481,nid007077,nid005208,nid005207,nid005879,nid005901
3
+ #SBATCH --nodes=32
4
+ #SBATCH --ntasks-per-node=1
5
+ #SBATCH --cpus-per-task=40
6
+ #SBATCH --mem=256G
7
+ #SBATCH -p standard-g
8
+ #SBATCH -t 48:00:00
9
+ #SBATCH --gpus-per-node=mi250:8
10
+ #SBATCH --exclusive=user
11
+ #SBATCH --hint=nomultithread
12
+ #SBATCH --account=project_462000119
13
+ #SBATCH -o logs/%j.out
14
+ #SBATCH -e logs/%j.err
15
+
16
+ VARIANT=4b284b28bc4seed2
17
+
18
+ # if run without sbatch, invoke here
19
+ if [ -z $SLURM_JOB_ID ]; then
20
+ mkdir -p logs
21
+ sbatch "$0"
22
+ exit
23
+ fi
24
+
25
+ set -euo pipefail
26
+
27
+ # symlink logs/latest.out and logs/latest.err
28
+ ln -f -s $SLURM_JOB_ID.out logs/latest.out
29
+ ln -f -s $SLURM_JOB_ID.err logs/latest.err
30
+
31
+ KILL_SWITCH_PATH=kill-switch-$VARIANT
32
+ CHECKPOINT_PATH=checkpoints_$VARIANT
33
+ TENSORBOARD_PATH=tensorboard_$VARIANT
34
+
35
+ # Data
36
+ VOCAB_FILE="gpt2/vocab.json"
37
+ MERGE_FILE="gpt2/merges.txt"
38
+
39
+ TRAIN_DATA_PATH=train28b.txt
40
+ # "train: 1.0 0:1 /scratch/project_462000119/data/c4_subsampled/gpt2tok_c4_en_28B_text_document"
41
+ VALID_DATA_PATH=val.txt
42
+ # "validation: 1.0 0:1 /scratch/project_462000119/data/c4_validation/gpt2tok_c4validation_rerun_text_document"
43
+
44
+ PP_SIZE=1
45
+ TP_SIZE=2
46
+
47
+ MICRO_BATCH_SIZE=2
48
+ GRADIENT_ACCUMULATION_STEPS=1
49
+ WORLD_SIZE=$((SLURM_GPUS_ON_NODE*SLURM_JOB_NUM_NODES))
50
+ GLOBAL_BATCH_SIZE=$((MICRO_BATCH_SIZE*WORLD_SIZE*GRADIENT_ACCUMULATION_STEPS))
51
+
52
+ # Model parameters
53
+ source model_params.sh
54
+ MODEL_PARAM=("${PARAM_4516M[@]}")
55
+ NHIDDEN=${MODEL_PARAM[0]}
56
+ FFN_HIDDEN_SIZE=${MODEL_PARAM[1]}
57
+ KV_SIZE=${MODEL_PARAM[2]}
58
+ NHEADS=${MODEL_PARAM[3]}
59
+ NLAYERS=${MODEL_PARAM[4]}
60
+ SEQ_LEN=2048
61
+
62
+ echo "Model parameters: d_model $NHIDDEN ffw_size $FFN_HIDDEN_SIZE kv_size $KV_SIZE n_heads $NHEADS n_layers $NLAYERS"
63
+
64
+ SAVE_INTERVAL=10000
65
+
66
+ # Tokens: 84_000_000_000
67
+ # -> Samples: 41_015_625.0
68
+ TRAIN_SAMPLES=41_015_625
69
+
70
+ OPTIMIZER_ARGS=" \
71
+ --optimizer adam \
72
+ --adam-beta1 0.9 \
73
+ --adam-beta2 0.95 \
74
+ --adam-eps 1e-8 \
75
+ --lr 2e-4 \
76
+ --min-lr 2e-5 \
77
+ --lr-decay-style cosine \
78
+ --lr-decay-samples $TRAIN_SAMPLES \
79
+ --lr-warmup-samples 410_156 \
80
+ --clip-grad 1.0 \
81
+ --weight-decay 1e-1 \
82
+ "
83
+
84
+ GPT_ARGS=" \
85
+ --num-layers $NLAYERS \
86
+ --hidden-size $NHIDDEN \
87
+ --num-attention-heads $NHEADS \
88
+ --kv-channels $KV_SIZE \
89
+ --ffn-hidden-size $FFN_HIDDEN_SIZE \
90
+ --seq-length $SEQ_LEN \
91
+ --max-position-embeddings $SEQ_LEN \
92
+ --micro-batch-size $MICRO_BATCH_SIZE \
93
+ --global-batch-size $GLOBAL_BATCH_SIZE \
94
+ --train-samples $TRAIN_SAMPLES \
95
+ --vocab-file $VOCAB_FILE \
96
+ --merge-file $MERGE_FILE \
97
+ --clip-grad 1.0 \
98
+ --kill-switch-path $KILL_SWITCH_PATH \
99
+ --bf16 \
100
+ $OPTIMIZER_ARGS \
101
+ "
102
+
103
+ OUTPUT_ARGS=" \
104
+ --log-interval 10 \
105
+ --save-interval $SAVE_INTERVAL \
106
+ --eval-interval 1000 \
107
+ --eval-iters 100 \
108
+ --tensorboard-dir $TENSORBOARD_PATH \
109
+ --tensorboard-queue-size 5 \
110
+ --log-timers-to-tensorboard \
111
+ --log-batch-size-to-tensorboard \
112
+ --log-validation-ppl-to-tensorboard \
113
+ "
114
+
115
+ ZERO_STAGE=0
116
+
117
+ mkdir -p ds_configs
118
+ DS_CONFIG_PATH="ds_configs/$SLURM_JOB_ID.json"
119
+
120
+ cat <<EOF > $DS_CONFIG_PATH
121
+ {
122
+ "train_micro_batch_size_per_gpu": $MICRO_BATCH_SIZE,
123
+ "train_batch_size": $GLOBAL_BATCH_SIZE,
124
+ "gradient_clipping": 1.0,
125
+ "zero_optimization": {
126
+ "stage": $ZERO_STAGE
127
+ },
128
+ "bf16": {
129
+ "enabled": true
130
+ },
131
+ "steps_per_print": 2000,
132
+ "wall_clock_breakdown": false
133
+ }
134
+ EOF
135
+
136
+ DEEPSPEED_ARGS=" \
137
+ --deepspeed \
138
+ --deepspeed_config $DS_CONFIG_PATH \
139
+ --zero-stage $ZERO_STAGE \
140
+ "
141
+
142
+ CMD=" \
143
+ Megatron-DeepSpeed/pretrain_gpt.py \
144
+ --tensor-model-parallel-size $TP_SIZE \
145
+ --pipeline-model-parallel-size $PP_SIZE \
146
+ $GPT_ARGS \
147
+ $OUTPUT_ARGS \
148
+ --save $CHECKPOINT_PATH \
149
+ --load $CHECKPOINT_PATH \
150
+ --train-weighted-split-paths-path $TRAIN_DATA_PATH \
151
+ --valid-weighted-split-paths-path $VALID_DATA_PATH \
152
+ --data-impl mmap \
153
+ --seed 2 \
154
+ $DEEPSPEED_ARGS \
155
+ "
156
+
157
+ echo $CMD
158
+
159
+ echo "START $SLURM_JOBID: $(date)"
160
+
161
+ # bash launch_srun.sh $CMD
162
+ srun --label launch.sh $CMD
163
+
164
+ echo "END $SLURM_JOBID: $(date)"
sbatch_4b284b28bc4seed3.sh ADDED
@@ -0,0 +1,164 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #!/bin/bash
2
+ #SBATCH --exclude=nid007571,nid007112,nid006774,nid007502,nid007506,nid007507,nid005145,nid006692,nid007218,nid007123,nid006124,nid006123,nid007496,nid007237,nid006852,nid007206,nid006947,nid007212,nid006977,nid007222,nid005444,nid007219,nid007493,nid007221,nid005300,nid005619,nid006118,nid005203,nid006113,nid006481,nid007077,nid005208,nid005207,nid005879,nid005901
3
+ #SBATCH --nodes=32
4
+ #SBATCH --ntasks-per-node=1
5
+ #SBATCH --cpus-per-task=40
6
+ #SBATCH --mem=256G
7
+ #SBATCH -p standard-g
8
+ #SBATCH -t 48:00:00
9
+ #SBATCH --gpus-per-node=mi250:8
10
+ #SBATCH --exclusive=user
11
+ #SBATCH --hint=nomultithread
12
+ #SBATCH --account=project_462000119
13
+ #SBATCH -o logs/%j.out
14
+ #SBATCH -e logs/%j.err
15
+
16
+ VARIANT=4b284b28bc4seed3
17
+
18
+ # if run without sbatch, invoke here
19
+ if [ -z $SLURM_JOB_ID ]; then
20
+ mkdir -p logs
21
+ sbatch "$0"
22
+ exit
23
+ fi
24
+
25
+ set -euo pipefail
26
+
27
+ # symlink logs/latest.out and logs/latest.err
28
+ ln -f -s $SLURM_JOB_ID.out logs/latest.out
29
+ ln -f -s $SLURM_JOB_ID.err logs/latest.err
30
+
31
+ KILL_SWITCH_PATH=kill-switch-$VARIANT
32
+ CHECKPOINT_PATH=checkpoints_$VARIANT
33
+ TENSORBOARD_PATH=tensorboard_$VARIANT
34
+
35
+ # Data
36
+ VOCAB_FILE="gpt2/vocab.json"
37
+ MERGE_FILE="gpt2/merges.txt"
38
+
39
+ TRAIN_DATA_PATH=train28b.txt
40
+ # "train: 1.0 0:1 /scratch/project_462000119/data/c4_subsampled/gpt2tok_c4_en_28B_text_document"
41
+ VALID_DATA_PATH=val.txt
42
+ # "validation: 1.0 0:1 /scratch/project_462000119/data/c4_validation/gpt2tok_c4validation_rerun_text_document"
43
+
44
+ PP_SIZE=1
45
+ TP_SIZE=2
46
+
47
+ MICRO_BATCH_SIZE=2
48
+ GRADIENT_ACCUMULATION_STEPS=1
49
+ WORLD_SIZE=$((SLURM_GPUS_ON_NODE*SLURM_JOB_NUM_NODES))
50
+ GLOBAL_BATCH_SIZE=$((MICRO_BATCH_SIZE*WORLD_SIZE*GRADIENT_ACCUMULATION_STEPS))
51
+
52
+ # Model parameters
53
+ source model_params.sh
54
+ MODEL_PARAM=("${PARAM_4516M[@]}")
55
+ NHIDDEN=${MODEL_PARAM[0]}
56
+ FFN_HIDDEN_SIZE=${MODEL_PARAM[1]}
57
+ KV_SIZE=${MODEL_PARAM[2]}
58
+ NHEADS=${MODEL_PARAM[3]}
59
+ NLAYERS=${MODEL_PARAM[4]}
60
+ SEQ_LEN=2048
61
+
62
+ echo "Model parameters: d_model $NHIDDEN ffw_size $FFN_HIDDEN_SIZE kv_size $KV_SIZE n_heads $NHEADS n_layers $NLAYERS"
63
+
64
+ SAVE_INTERVAL=10000
65
+
66
+ # Tokens: 84_000_000_000
67
+ # -> Samples: 41_015_625.0
68
+ TRAIN_SAMPLES=41_015_625
69
+
70
+ OPTIMIZER_ARGS=" \
71
+ --optimizer adam \
72
+ --adam-beta1 0.9 \
73
+ --adam-beta2 0.95 \
74
+ --adam-eps 1e-8 \
75
+ --lr 2e-4 \
76
+ --min-lr 2e-5 \
77
+ --lr-decay-style cosine \
78
+ --lr-decay-samples $TRAIN_SAMPLES \
79
+ --lr-warmup-samples 410_156 \
80
+ --clip-grad 1.0 \
81
+ --weight-decay 1e-1 \
82
+ "
83
+
84
+ GPT_ARGS=" \
85
+ --num-layers $NLAYERS \
86
+ --hidden-size $NHIDDEN \
87
+ --num-attention-heads $NHEADS \
88
+ --kv-channels $KV_SIZE \
89
+ --ffn-hidden-size $FFN_HIDDEN_SIZE \
90
+ --seq-length $SEQ_LEN \
91
+ --max-position-embeddings $SEQ_LEN \
92
+ --micro-batch-size $MICRO_BATCH_SIZE \
93
+ --global-batch-size $GLOBAL_BATCH_SIZE \
94
+ --train-samples $TRAIN_SAMPLES \
95
+ --vocab-file $VOCAB_FILE \
96
+ --merge-file $MERGE_FILE \
97
+ --clip-grad 1.0 \
98
+ --kill-switch-path $KILL_SWITCH_PATH \
99
+ --bf16 \
100
+ $OPTIMIZER_ARGS \
101
+ "
102
+
103
+ OUTPUT_ARGS=" \
104
+ --log-interval 10 \
105
+ --save-interval $SAVE_INTERVAL \
106
+ --eval-interval 1000 \
107
+ --eval-iters 100 \
108
+ --tensorboard-dir $TENSORBOARD_PATH \
109
+ --tensorboard-queue-size 5 \
110
+ --log-timers-to-tensorboard \
111
+ --log-batch-size-to-tensorboard \
112
+ --log-validation-ppl-to-tensorboard \
113
+ "
114
+
115
+ ZERO_STAGE=0
116
+
117
+ mkdir -p ds_configs
118
+ DS_CONFIG_PATH="ds_configs/$SLURM_JOB_ID.json"
119
+
120
+ cat <<EOF > $DS_CONFIG_PATH
121
+ {
122
+ "train_micro_batch_size_per_gpu": $MICRO_BATCH_SIZE,
123
+ "train_batch_size": $GLOBAL_BATCH_SIZE,
124
+ "gradient_clipping": 1.0,
125
+ "zero_optimization": {
126
+ "stage": $ZERO_STAGE
127
+ },
128
+ "bf16": {
129
+ "enabled": true
130
+ },
131
+ "steps_per_print": 2000,
132
+ "wall_clock_breakdown": false
133
+ }
134
+ EOF
135
+
136
+ DEEPSPEED_ARGS=" \
137
+ --deepspeed \
138
+ --deepspeed_config $DS_CONFIG_PATH \
139
+ --zero-stage $ZERO_STAGE \
140
+ "
141
+
142
+ CMD=" \
143
+ Megatron-DeepSpeed/pretrain_gpt.py \
144
+ --tensor-model-parallel-size $TP_SIZE \
145
+ --pipeline-model-parallel-size $PP_SIZE \
146
+ $GPT_ARGS \
147
+ $OUTPUT_ARGS \
148
+ --save $CHECKPOINT_PATH \
149
+ --load $CHECKPOINT_PATH \
150
+ --train-weighted-split-paths-path $TRAIN_DATA_PATH \
151
+ --valid-weighted-split-paths-path $VALID_DATA_PATH \
152
+ --data-impl mmap \
153
+ --seed 3 \
154
+ $DEEPSPEED_ARGS \
155
+ "
156
+
157
+ echo $CMD
158
+
159
+ echo "START $SLURM_JOBID: $(date)"
160
+
161
+ # bash launch_srun.sh $CMD
162
+ srun --label launch.sh $CMD
163
+
164
+ echo "END $SLURM_JOBID: $(date)"
sbatch_4b284b28bc4seed4.sh ADDED
@@ -0,0 +1,164 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #!/bin/bash
2
+ #SBATCH --exclude=nid007571,nid007112,nid006774,nid007502,nid007506,nid007507,nid005145,nid006692,nid007218,nid007123,nid006124,nid006123,nid007496,nid007237,nid006852,nid007206,nid006947,nid007212,nid006977,nid007222,nid005444,nid007219,nid007493,nid007221,nid005300,nid005619,nid006118,nid005203,nid006113,nid006481,nid007077,nid005208,nid005207,nid005879,nid005901
3
+ #SBATCH --nodes=32
4
+ #SBATCH --ntasks-per-node=1
5
+ #SBATCH --cpus-per-task=40
6
+ #SBATCH --mem=256G
7
+ #SBATCH -p standard-g
8
+ #SBATCH -t 48:00:00
9
+ #SBATCH --gpus-per-node=mi250:8
10
+ #SBATCH --exclusive=user
11
+ #SBATCH --hint=nomultithread
12
+ #SBATCH --account=project_462000119
13
+ #SBATCH -o logs/%j.out
14
+ #SBATCH -e logs/%j.err
15
+
16
+ VARIANT=4b284b28bc4seed4
17
+
18
+ # if run without sbatch, invoke here
19
+ if [ -z $SLURM_JOB_ID ]; then
20
+ mkdir -p logs
21
+ sbatch "$0"
22
+ exit
23
+ fi
24
+
25
+ set -euo pipefail
26
+
27
+ # symlink logs/latest.out and logs/latest.err
28
+ ln -f -s $SLURM_JOB_ID.out logs/latest.out
29
+ ln -f -s $SLURM_JOB_ID.err logs/latest.err
30
+
31
+ KILL_SWITCH_PATH=kill-switch-$VARIANT
32
+ CHECKPOINT_PATH=checkpoints_$VARIANT
33
+ TENSORBOARD_PATH=tensorboard_$VARIANT
34
+
35
+ # Data
36
+ VOCAB_FILE="gpt2/vocab.json"
37
+ MERGE_FILE="gpt2/merges.txt"
38
+
39
+ TRAIN_DATA_PATH=train28b.txt
40
+ # "train: 1.0 0:1 /scratch/project_462000119/data/c4_subsampled/gpt2tok_c4_en_28B_text_document"
41
+ VALID_DATA_PATH=val.txt
42
+ # "validation: 1.0 0:1 /scratch/project_462000119/data/c4_validation/gpt2tok_c4validation_rerun_text_document"
43
+
44
+ PP_SIZE=1
45
+ TP_SIZE=2
46
+
47
+ MICRO_BATCH_SIZE=2
48
+ GRADIENT_ACCUMULATION_STEPS=1
49
+ WORLD_SIZE=$((SLURM_GPUS_ON_NODE*SLURM_JOB_NUM_NODES))
50
+ GLOBAL_BATCH_SIZE=$((MICRO_BATCH_SIZE*WORLD_SIZE*GRADIENT_ACCUMULATION_STEPS))
51
+
52
+ # Model parameters
53
+ source model_params.sh
54
+ MODEL_PARAM=("${PARAM_4516M[@]}")
55
+ NHIDDEN=${MODEL_PARAM[0]}
56
+ FFN_HIDDEN_SIZE=${MODEL_PARAM[1]}
57
+ KV_SIZE=${MODEL_PARAM[2]}
58
+ NHEADS=${MODEL_PARAM[3]}
59
+ NLAYERS=${MODEL_PARAM[4]}
60
+ SEQ_LEN=2048
61
+
62
+ echo "Model parameters: d_model $NHIDDEN ffw_size $FFN_HIDDEN_SIZE kv_size $KV_SIZE n_heads $NHEADS n_layers $NLAYERS"
63
+
64
+ SAVE_INTERVAL=10000
65
+
66
+ # Tokens: 84_000_000_000
67
+ # -> Samples: 41_015_625.0
68
+ TRAIN_SAMPLES=41_015_625
69
+
70
+ OPTIMIZER_ARGS=" \
71
+ --optimizer adam \
72
+ --adam-beta1 0.9 \
73
+ --adam-beta2 0.95 \
74
+ --adam-eps 1e-8 \
75
+ --lr 2e-4 \
76
+ --min-lr 2e-5 \
77
+ --lr-decay-style cosine \
78
+ --lr-decay-samples $TRAIN_SAMPLES \
79
+ --lr-warmup-samples 410_156 \
80
+ --clip-grad 1.0 \
81
+ --weight-decay 1e-1 \
82
+ "
83
+
84
+ GPT_ARGS=" \
85
+ --num-layers $NLAYERS \
86
+ --hidden-size $NHIDDEN \
87
+ --num-attention-heads $NHEADS \
88
+ --kv-channels $KV_SIZE \
89
+ --ffn-hidden-size $FFN_HIDDEN_SIZE \
90
+ --seq-length $SEQ_LEN \
91
+ --max-position-embeddings $SEQ_LEN \
92
+ --micro-batch-size $MICRO_BATCH_SIZE \
93
+ --global-batch-size $GLOBAL_BATCH_SIZE \
94
+ --train-samples $TRAIN_SAMPLES \
95
+ --vocab-file $VOCAB_FILE \
96
+ --merge-file $MERGE_FILE \
97
+ --clip-grad 1.0 \
98
+ --kill-switch-path $KILL_SWITCH_PATH \
99
+ --bf16 \
100
+ $OPTIMIZER_ARGS \
101
+ "
102
+
103
+ OUTPUT_ARGS=" \
104
+ --log-interval 10 \
105
+ --save-interval $SAVE_INTERVAL \
106
+ --eval-interval 1000 \
107
+ --eval-iters 100 \
108
+ --tensorboard-dir $TENSORBOARD_PATH \
109
+ --tensorboard-queue-size 5 \
110
+ --log-timers-to-tensorboard \
111
+ --log-batch-size-to-tensorboard \
112
+ --log-validation-ppl-to-tensorboard \
113
+ "
114
+
115
+ ZERO_STAGE=0
116
+
117
+ mkdir -p ds_configs
118
+ DS_CONFIG_PATH="ds_configs/$SLURM_JOB_ID.json"
119
+
120
+ cat <<EOF > $DS_CONFIG_PATH
121
+ {
122
+ "train_micro_batch_size_per_gpu": $MICRO_BATCH_SIZE,
123
+ "train_batch_size": $GLOBAL_BATCH_SIZE,
124
+ "gradient_clipping": 1.0,
125
+ "zero_optimization": {
126
+ "stage": $ZERO_STAGE
127
+ },
128
+ "bf16": {
129
+ "enabled": true
130
+ },
131
+ "steps_per_print": 2000,
132
+ "wall_clock_breakdown": false
133
+ }
134
+ EOF
135
+
136
+ DEEPSPEED_ARGS=" \
137
+ --deepspeed \
138
+ --deepspeed_config $DS_CONFIG_PATH \
139
+ --zero-stage $ZERO_STAGE \
140
+ "
141
+
142
+ CMD=" \
143
+ Megatron-DeepSpeed/pretrain_gpt.py \
144
+ --tensor-model-parallel-size $TP_SIZE \
145
+ --pipeline-model-parallel-size $PP_SIZE \
146
+ $GPT_ARGS \
147
+ $OUTPUT_ARGS \
148
+ --save $CHECKPOINT_PATH \
149
+ --load $CHECKPOINT_PATH \
150
+ --train-weighted-split-paths-path $TRAIN_DATA_PATH \
151
+ --valid-weighted-split-paths-path $VALID_DATA_PATH \
152
+ --data-impl mmap \
153
+ --seed 4 \
154
+ $DEEPSPEED_ARGS \
155
+ "
156
+
157
+ echo $CMD
158
+
159
+ echo "START $SLURM_JOBID: $(date)"
160
+
161
+ # bash launch_srun.sh $CMD
162
+ srun --label launch.sh $CMD
163
+
164
+ echo "END $SLURM_JOBID: $(date)"
sbatch_4b284b42bc4seed1.sh ADDED
@@ -0,0 +1,164 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #!/bin/bash
2
+ #SBATCH --exclude=nid007571,nid007112,nid006774,nid007502,nid007506,nid007507,nid005145,nid006692,nid007218,nid007123,nid006124,nid006123,nid007496,nid007237,nid006852,nid007206,nid006947,nid007212,nid006977,nid007222,nid005444,nid007219,nid007493,nid007221,nid005300,nid005619,nid006118,nid005203,nid006113,nid006481,nid007077,nid005208,nid005207,nid005879,nid005901
3
+ #SBATCH --nodes=32
4
+ #SBATCH --ntasks-per-node=1
5
+ #SBATCH --cpus-per-task=40
6
+ #SBATCH --mem=256G
7
+ #SBATCH -p standard-g
8
+ #SBATCH -t 48:00:00
9
+ #SBATCH --gpus-per-node=mi250:8
10
+ #SBATCH --exclusive=user
11
+ #SBATCH --hint=nomultithread
12
+ #SBATCH --account=project_462000119
13
+ #SBATCH -o logs/%j.out
14
+ #SBATCH -e logs/%j.err
15
+
16
+ VARIANT=4b284b42bc4seed1
17
+
18
+ # if run without sbatch, invoke here
19
+ if [ -z $SLURM_JOB_ID ]; then
20
+ mkdir -p logs
21
+ sbatch "$0"
22
+ exit
23
+ fi
24
+
25
+ set -euo pipefail
26
+
27
+ # symlink logs/latest.out and logs/latest.err
28
+ ln -f -s $SLURM_JOB_ID.out logs/latest.out
29
+ ln -f -s $SLURM_JOB_ID.err logs/latest.err
30
+
31
+ KILL_SWITCH_PATH=kill-switch-$VARIANT
32
+ CHECKPOINT_PATH=checkpoints_$VARIANT
33
+ TENSORBOARD_PATH=tensorboard_$VARIANT
34
+
35
+ # Data
36
+ VOCAB_FILE="gpt2/vocab.json"
37
+ MERGE_FILE="gpt2/merges.txt"
38
+
39
+ TRAIN_DATA_PATH=train42b.txt
40
+ # "train: 1.0 0:1 /scratch/project_462000119/data/c4_subsampled/gpt2tok_c4_en_42B_text_document"
41
+ VALID_DATA_PATH=val.txt
42
+ # "validation: 1.0 0:1 /scratch/project_462000119/data/c4_validation/gpt2tok_c4validation_rerun_text_document"
43
+
44
+ PP_SIZE=1
45
+ TP_SIZE=2
46
+
47
+ MICRO_BATCH_SIZE=2
48
+ GRADIENT_ACCUMULATION_STEPS=1
49
+ WORLD_SIZE=$((SLURM_GPUS_ON_NODE*SLURM_JOB_NUM_NODES))
50
+ GLOBAL_BATCH_SIZE=$((MICRO_BATCH_SIZE*WORLD_SIZE*GRADIENT_ACCUMULATION_STEPS))
51
+
52
+ # Model parameters
53
+ source model_params.sh
54
+ MODEL_PARAM=("${PARAM_4516M[@]}")
55
+ NHIDDEN=${MODEL_PARAM[0]}
56
+ FFN_HIDDEN_SIZE=${MODEL_PARAM[1]}
57
+ KV_SIZE=${MODEL_PARAM[2]}
58
+ NHEADS=${MODEL_PARAM[3]}
59
+ NLAYERS=${MODEL_PARAM[4]}
60
+ SEQ_LEN=2048
61
+
62
+ echo "Model parameters: d_model $NHIDDEN ffw_size $FFN_HIDDEN_SIZE kv_size $KV_SIZE n_heads $NHEADS n_layers $NLAYERS"
63
+
64
+ SAVE_INTERVAL=10000
65
+
66
+ # Tokens: 84_000_000_000
67
+ # -> Samples: 41_015_625.0
68
+ TRAIN_SAMPLES=41_015_625
69
+
70
+ OPTIMIZER_ARGS=" \
71
+ --optimizer adam \
72
+ --adam-beta1 0.9 \
73
+ --adam-beta2 0.95 \
74
+ --adam-eps 1e-8 \
75
+ --lr 2e-4 \
76
+ --min-lr 2e-5 \
77
+ --lr-decay-style cosine \
78
+ --lr-decay-samples $TRAIN_SAMPLES \
79
+ --lr-warmup-samples 410_156 \
80
+ --clip-grad 1.0 \
81
+ --weight-decay 1e-1 \
82
+ "
83
+
84
+ GPT_ARGS=" \
85
+ --num-layers $NLAYERS \
86
+ --hidden-size $NHIDDEN \
87
+ --num-attention-heads $NHEADS \
88
+ --kv-channels $KV_SIZE \
89
+ --ffn-hidden-size $FFN_HIDDEN_SIZE \
90
+ --seq-length $SEQ_LEN \
91
+ --max-position-embeddings $SEQ_LEN \
92
+ --micro-batch-size $MICRO_BATCH_SIZE \
93
+ --global-batch-size $GLOBAL_BATCH_SIZE \
94
+ --train-samples $TRAIN_SAMPLES \
95
+ --vocab-file $VOCAB_FILE \
96
+ --merge-file $MERGE_FILE \
97
+ --clip-grad 1.0 \
98
+ --kill-switch-path $KILL_SWITCH_PATH \
99
+ --bf16 \
100
+ $OPTIMIZER_ARGS \
101
+ "
102
+
103
+ OUTPUT_ARGS=" \
104
+ --log-interval 10 \
105
+ --save-interval $SAVE_INTERVAL \
106
+ --eval-interval 1000 \
107
+ --eval-iters 100 \
108
+ --tensorboard-dir $TENSORBOARD_PATH \
109
+ --tensorboard-queue-size 5 \
110
+ --log-timers-to-tensorboard \
111
+ --log-batch-size-to-tensorboard \
112
+ --log-validation-ppl-to-tensorboard \
113
+ "
114
+
115
+ ZERO_STAGE=0
116
+
117
+ mkdir -p ds_configs
118
+ DS_CONFIG_PATH="ds_configs/$SLURM_JOB_ID.json"
119
+
120
+ cat <<EOF > $DS_CONFIG_PATH
121
+ {
122
+ "train_micro_batch_size_per_gpu": $MICRO_BATCH_SIZE,
123
+ "train_batch_size": $GLOBAL_BATCH_SIZE,
124
+ "gradient_clipping": 1.0,
125
+ "zero_optimization": {
126
+ "stage": $ZERO_STAGE
127
+ },
128
+ "bf16": {
129
+ "enabled": true
130
+ },
131
+ "steps_per_print": 2000,
132
+ "wall_clock_breakdown": false
133
+ }
134
+ EOF
135
+
136
+ DEEPSPEED_ARGS=" \
137
+ --deepspeed \
138
+ --deepspeed_config $DS_CONFIG_PATH \
139
+ --zero-stage $ZERO_STAGE \
140
+ "
141
+
142
+ CMD=" \
143
+ Megatron-DeepSpeed/pretrain_gpt.py \
144
+ --tensor-model-parallel-size $TP_SIZE \
145
+ --pipeline-model-parallel-size $PP_SIZE \
146
+ $GPT_ARGS \
147
+ $OUTPUT_ARGS \
148
+ --save $CHECKPOINT_PATH \
149
+ --load $CHECKPOINT_PATH \
150
+ --train-weighted-split-paths-path $TRAIN_DATA_PATH \
151
+ --valid-weighted-split-paths-path $VALID_DATA_PATH \
152
+ --data-impl mmap \
153
+ --seed 1 \
154
+ $DEEPSPEED_ARGS \
155
+ "
156
+
157
+ echo $CMD
158
+
159
+ echo "START $SLURM_JOBID: $(date)"
160
+
161
+ # bash launch_srun.sh $CMD
162
+ srun --label launch.sh $CMD
163
+
164
+ echo "END $SLURM_JOBID: $(date)"
sbatch_4b284b42bc4seed2.sh ADDED
@@ -0,0 +1,164 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #!/bin/bash
2
+ #SBATCH --exclude=nid007571,nid007112,nid006774,nid007502,nid007506,nid007507,nid005145,nid006692,nid007218,nid007123,nid006124,nid006123,nid007496,nid007237,nid006852,nid007206,nid006947,nid007212,nid006977,nid007222,nid005444,nid007219,nid007493,nid007221,nid005300,nid005619,nid006118,nid005203,nid006113,nid006481,nid007077,nid005208,nid005207,nid005879,nid005901
3
+ #SBATCH --nodes=32
4
+ #SBATCH --ntasks-per-node=1
5
+ #SBATCH --cpus-per-task=40
6
+ #SBATCH --mem=256G
7
+ #SBATCH -p standard-g
8
+ #SBATCH -t 48:00:00
9
+ #SBATCH --gpus-per-node=mi250:8
10
+ #SBATCH --exclusive=user
11
+ #SBATCH --hint=nomultithread
12
+ #SBATCH --account=project_462000119
13
+ #SBATCH -o logs/%j.out
14
+ #SBATCH -e logs/%j.err
15
+
16
+ VARIANT=4b284b42bc4seed2
17
+
18
+ # if run without sbatch, invoke here
19
+ if [ -z $SLURM_JOB_ID ]; then
20
+ mkdir -p logs
21
+ sbatch "$0"
22
+ exit
23
+ fi
24
+
25
+ set -euo pipefail
26
+
27
+ # symlink logs/latest.out and logs/latest.err
28
+ ln -f -s $SLURM_JOB_ID.out logs/latest.out
29
+ ln -f -s $SLURM_JOB_ID.err logs/latest.err
30
+
31
+ KILL_SWITCH_PATH=kill-switch-$VARIANT
32
+ CHECKPOINT_PATH=checkpoints_$VARIANT
33
+ TENSORBOARD_PATH=tensorboard_$VARIANT
34
+
35
+ # Data
36
+ VOCAB_FILE="gpt2/vocab.json"
37
+ MERGE_FILE="gpt2/merges.txt"
38
+
39
+ TRAIN_DATA_PATH=train42b.txt
40
+ # "train: 1.0 0:1 /scratch/project_462000119/data/c4_subsampled/gpt2tok_c4_en_42B_text_document"
41
+ VALID_DATA_PATH=val.txt
42
+ # "validation: 1.0 0:1 /scratch/project_462000119/data/c4_validation/gpt2tok_c4validation_rerun_text_document"
43
+
44
+ PP_SIZE=1
45
+ TP_SIZE=2
46
+
47
+ MICRO_BATCH_SIZE=2
48
+ GRADIENT_ACCUMULATION_STEPS=1
49
+ WORLD_SIZE=$((SLURM_GPUS_ON_NODE*SLURM_JOB_NUM_NODES))
50
+ GLOBAL_BATCH_SIZE=$((MICRO_BATCH_SIZE*WORLD_SIZE*GRADIENT_ACCUMULATION_STEPS))
51
+
52
+ # Model parameters
53
+ source model_params.sh
54
+ MODEL_PARAM=("${PARAM_4516M[@]}")
55
+ NHIDDEN=${MODEL_PARAM[0]}
56
+ FFN_HIDDEN_SIZE=${MODEL_PARAM[1]}
57
+ KV_SIZE=${MODEL_PARAM[2]}
58
+ NHEADS=${MODEL_PARAM[3]}
59
+ NLAYERS=${MODEL_PARAM[4]}
60
+ SEQ_LEN=2048
61
+
62
+ echo "Model parameters: d_model $NHIDDEN ffw_size $FFN_HIDDEN_SIZE kv_size $KV_SIZE n_heads $NHEADS n_layers $NLAYERS"
63
+
64
+ SAVE_INTERVAL=10000
65
+
66
+ # Tokens: 84_000_000_000
67
+ # -> Samples: 41_015_625.0
68
+ TRAIN_SAMPLES=41_015_625
69
+
70
+ OPTIMIZER_ARGS=" \
71
+ --optimizer adam \
72
+ --adam-beta1 0.9 \
73
+ --adam-beta2 0.95 \
74
+ --adam-eps 1e-8 \
75
+ --lr 2e-4 \
76
+ --min-lr 2e-5 \
77
+ --lr-decay-style cosine \
78
+ --lr-decay-samples $TRAIN_SAMPLES \
79
+ --lr-warmup-samples 410_156 \
80
+ --clip-grad 1.0 \
81
+ --weight-decay 1e-1 \
82
+ "
83
+
84
+ GPT_ARGS=" \
85
+ --num-layers $NLAYERS \
86
+ --hidden-size $NHIDDEN \
87
+ --num-attention-heads $NHEADS \
88
+ --kv-channels $KV_SIZE \
89
+ --ffn-hidden-size $FFN_HIDDEN_SIZE \
90
+ --seq-length $SEQ_LEN \
91
+ --max-position-embeddings $SEQ_LEN \
92
+ --micro-batch-size $MICRO_BATCH_SIZE \
93
+ --global-batch-size $GLOBAL_BATCH_SIZE \
94
+ --train-samples $TRAIN_SAMPLES \
95
+ --vocab-file $VOCAB_FILE \
96
+ --merge-file $MERGE_FILE \
97
+ --clip-grad 1.0 \
98
+ --kill-switch-path $KILL_SWITCH_PATH \
99
+ --bf16 \
100
+ $OPTIMIZER_ARGS \
101
+ "
102
+
103
+ OUTPUT_ARGS=" \
104
+ --log-interval 10 \
105
+ --save-interval $SAVE_INTERVAL \
106
+ --eval-interval 1000 \
107
+ --eval-iters 100 \
108
+ --tensorboard-dir $TENSORBOARD_PATH \
109
+ --tensorboard-queue-size 5 \
110
+ --log-timers-to-tensorboard \
111
+ --log-batch-size-to-tensorboard \
112
+ --log-validation-ppl-to-tensorboard \
113
+ "
114
+
115
+ ZERO_STAGE=0
116
+
117
+ mkdir -p ds_configs
118
+ DS_CONFIG_PATH="ds_configs/$SLURM_JOB_ID.json"
119
+
120
+ cat <<EOF > $DS_CONFIG_PATH
121
+ {
122
+ "train_micro_batch_size_per_gpu": $MICRO_BATCH_SIZE,
123
+ "train_batch_size": $GLOBAL_BATCH_SIZE,
124
+ "gradient_clipping": 1.0,
125
+ "zero_optimization": {
126
+ "stage": $ZERO_STAGE
127
+ },
128
+ "bf16": {
129
+ "enabled": true
130
+ },
131
+ "steps_per_print": 2000,
132
+ "wall_clock_breakdown": false
133
+ }
134
+ EOF
135
+
136
+ DEEPSPEED_ARGS=" \
137
+ --deepspeed \
138
+ --deepspeed_config $DS_CONFIG_PATH \
139
+ --zero-stage $ZERO_STAGE \
140
+ "
141
+
142
+ CMD=" \
143
+ Megatron-DeepSpeed/pretrain_gpt.py \
144
+ --tensor-model-parallel-size $TP_SIZE \
145
+ --pipeline-model-parallel-size $PP_SIZE \
146
+ $GPT_ARGS \
147
+ $OUTPUT_ARGS \
148
+ --save $CHECKPOINT_PATH \
149
+ --load $CHECKPOINT_PATH \
150
+ --train-weighted-split-paths-path $TRAIN_DATA_PATH \
151
+ --valid-weighted-split-paths-path $VALID_DATA_PATH \
152
+ --data-impl mmap \
153
+ --seed 2 \
154
+ $DEEPSPEED_ARGS \
155
+ "
156
+
157
+ echo $CMD
158
+
159
+ echo "START $SLURM_JOBID: $(date)"
160
+
161
+ # bash launch_srun.sh $CMD
162
+ srun --label launch.sh $CMD
163
+
164
+ echo "END $SLURM_JOBID: $(date)"
sbatch_4b284b42bc4seed3.sh ADDED
@@ -0,0 +1,164 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #!/bin/bash
2
+ #SBATCH --exclude=nid007571,nid007112,nid006774,nid007502,nid007506,nid007507,nid005145,nid006692,nid007218,nid007123,nid006124,nid006123,nid007496,nid007237,nid006852,nid007206,nid006947,nid007212,nid006977,nid007222,nid005444,nid007219,nid007493,nid007221,nid005300,nid005619,nid006118,nid005203,nid006113,nid006481,nid007077,nid005208,nid005207,nid005879,nid005901
3
+ #SBATCH --nodes=32
4
+ #SBATCH --ntasks-per-node=1
5
+ #SBATCH --cpus-per-task=40
6
+ #SBATCH --mem=256G
7
+ #SBATCH -p standard-g
8
+ #SBATCH -t 48:00:00
9
+ #SBATCH --gpus-per-node=mi250:8
10
+ #SBATCH --exclusive=user
11
+ #SBATCH --hint=nomultithread
12
+ #SBATCH --account=project_462000119
13
+ #SBATCH -o logs/%j.out
14
+ #SBATCH -e logs/%j.err
15
+
16
+ VARIANT=4b284b42bc4seed3
17
+
18
+ # if run without sbatch, invoke here
19
+ if [ -z $SLURM_JOB_ID ]; then
20
+ mkdir -p logs
21
+ sbatch "$0"
22
+ exit
23
+ fi
24
+
25
+ set -euo pipefail
26
+
27
+ # symlink logs/latest.out and logs/latest.err
28
+ ln -f -s $SLURM_JOB_ID.out logs/latest.out
29
+ ln -f -s $SLURM_JOB_ID.err logs/latest.err
30
+
31
+ KILL_SWITCH_PATH=kill-switch-$VARIANT
32
+ CHECKPOINT_PATH=checkpoints_$VARIANT
33
+ TENSORBOARD_PATH=tensorboard_$VARIANT
34
+
35
+ # Data
36
+ VOCAB_FILE="gpt2/vocab.json"
37
+ MERGE_FILE="gpt2/merges.txt"
38
+
39
+ TRAIN_DATA_PATH=train42b.txt
40
+ # "train: 1.0 0:1 /scratch/project_462000119/data/c4_subsampled/gpt2tok_c4_en_42B_text_document"
41
+ VALID_DATA_PATH=val.txt
42
+ # "validation: 1.0 0:1 /scratch/project_462000119/data/c4_validation/gpt2tok_c4validation_rerun_text_document"
43
+
44
+ PP_SIZE=1
45
+ TP_SIZE=2
46
+
47
+ MICRO_BATCH_SIZE=2
48
+ GRADIENT_ACCUMULATION_STEPS=1
49
+ WORLD_SIZE=$((SLURM_GPUS_ON_NODE*SLURM_JOB_NUM_NODES))
50
+ GLOBAL_BATCH_SIZE=$((MICRO_BATCH_SIZE*WORLD_SIZE*GRADIENT_ACCUMULATION_STEPS))
51
+
52
+ # Model parameters
53
+ source model_params.sh
54
+ MODEL_PARAM=("${PARAM_4516M[@]}")
55
+ NHIDDEN=${MODEL_PARAM[0]}
56
+ FFN_HIDDEN_SIZE=${MODEL_PARAM[1]}
57
+ KV_SIZE=${MODEL_PARAM[2]}
58
+ NHEADS=${MODEL_PARAM[3]}
59
+ NLAYERS=${MODEL_PARAM[4]}
60
+ SEQ_LEN=2048
61
+
62
+ echo "Model parameters: d_model $NHIDDEN ffw_size $FFN_HIDDEN_SIZE kv_size $KV_SIZE n_heads $NHEADS n_layers $NLAYERS"
63
+
64
+ SAVE_INTERVAL=10000
65
+
66
+ # Tokens: 84_000_000_000
67
+ # -> Samples: 41_015_625.0
68
+ TRAIN_SAMPLES=41_015_625
69
+
70
+ OPTIMIZER_ARGS=" \
71
+ --optimizer adam \
72
+ --adam-beta1 0.9 \
73
+ --adam-beta2 0.95 \
74
+ --adam-eps 1e-8 \
75
+ --lr 2e-4 \
76
+ --min-lr 2e-5 \
77
+ --lr-decay-style cosine \
78
+ --lr-decay-samples $TRAIN_SAMPLES \
79
+ --lr-warmup-samples 410_156 \
80
+ --clip-grad 1.0 \
81
+ --weight-decay 1e-1 \
82
+ "
83
+
84
+ GPT_ARGS=" \
85
+ --num-layers $NLAYERS \
86
+ --hidden-size $NHIDDEN \
87
+ --num-attention-heads $NHEADS \
88
+ --kv-channels $KV_SIZE \
89
+ --ffn-hidden-size $FFN_HIDDEN_SIZE \
90
+ --seq-length $SEQ_LEN \
91
+ --max-position-embeddings $SEQ_LEN \
92
+ --micro-batch-size $MICRO_BATCH_SIZE \
93
+ --global-batch-size $GLOBAL_BATCH_SIZE \
94
+ --train-samples $TRAIN_SAMPLES \
95
+ --vocab-file $VOCAB_FILE \
96
+ --merge-file $MERGE_FILE \
97
+ --clip-grad 1.0 \
98
+ --kill-switch-path $KILL_SWITCH_PATH \
99
+ --bf16 \
100
+ $OPTIMIZER_ARGS \
101
+ "
102
+
103
+ OUTPUT_ARGS=" \
104
+ --log-interval 10 \
105
+ --save-interval $SAVE_INTERVAL \
106
+ --eval-interval 1000 \
107
+ --eval-iters 100 \
108
+ --tensorboard-dir $TENSORBOARD_PATH \
109
+ --tensorboard-queue-size 5 \
110
+ --log-timers-to-tensorboard \
111
+ --log-batch-size-to-tensorboard \
112
+ --log-validation-ppl-to-tensorboard \
113
+ "
114
+
115
+ ZERO_STAGE=0
116
+
117
+ mkdir -p ds_configs
118
+ DS_CONFIG_PATH="ds_configs/$SLURM_JOB_ID.json"
119
+
120
+ cat <<EOF > $DS_CONFIG_PATH
121
+ {
122
+ "train_micro_batch_size_per_gpu": $MICRO_BATCH_SIZE,
123
+ "train_batch_size": $GLOBAL_BATCH_SIZE,
124
+ "gradient_clipping": 1.0,
125
+ "zero_optimization": {
126
+ "stage": $ZERO_STAGE
127
+ },
128
+ "bf16": {
129
+ "enabled": true
130
+ },
131
+ "steps_per_print": 2000,
132
+ "wall_clock_breakdown": false
133
+ }
134
+ EOF
135
+
136
+ DEEPSPEED_ARGS=" \
137
+ --deepspeed \
138
+ --deepspeed_config $DS_CONFIG_PATH \
139
+ --zero-stage $ZERO_STAGE \
140
+ "
141
+
142
+ CMD=" \
143
+ Megatron-DeepSpeed/pretrain_gpt.py \
144
+ --tensor-model-parallel-size $TP_SIZE \
145
+ --pipeline-model-parallel-size $PP_SIZE \
146
+ $GPT_ARGS \
147
+ $OUTPUT_ARGS \
148
+ --save $CHECKPOINT_PATH \
149
+ --load $CHECKPOINT_PATH \
150
+ --train-weighted-split-paths-path $TRAIN_DATA_PATH \
151
+ --valid-weighted-split-paths-path $VALID_DATA_PATH \
152
+ --data-impl mmap \
153
+ --seed 3 \
154
+ $DEEPSPEED_ARGS \
155
+ "
156
+
157
+ echo $CMD
158
+
159
+ echo "START $SLURM_JOBID: $(date)"
160
+
161
+ # bash launch_srun.sh $CMD
162
+ srun --label launch.sh $CMD
163
+
164
+ echo "END $SLURM_JOBID: $(date)"
sbatch_4b284b42bc4seed4.sh ADDED
@@ -0,0 +1,164 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #!/bin/bash
2
+ #SBATCH --exclude=nid007571,nid007112,nid006774,nid007502,nid007506,nid007507,nid005145,nid006692,nid007218,nid007123,nid006124,nid006123,nid007496,nid007237,nid006852,nid007206,nid006947,nid007212,nid006977,nid007222,nid005444,nid007219,nid007493,nid007221,nid005300,nid005619,nid006118,nid005203,nid006113,nid006481,nid007077,nid005208,nid005207,nid005879,nid005901
3
+ #SBATCH --nodes=32
4
+ #SBATCH --ntasks-per-node=1
5
+ #SBATCH --cpus-per-task=40
6
+ #SBATCH --mem=256G
7
+ #SBATCH -p standard-g
8
+ #SBATCH -t 48:00:00
9
+ #SBATCH --gpus-per-node=mi250:8
10
+ #SBATCH --exclusive=user
11
+ #SBATCH --hint=nomultithread
12
+ #SBATCH --account=project_462000119
13
+ #SBATCH -o logs/%j.out
14
+ #SBATCH -e logs/%j.err
15
+
16
+ VARIANT=4b284b42bc4seed4
17
+
18
+ # if run without sbatch, invoke here
19
+ if [ -z $SLURM_JOB_ID ]; then
20
+ mkdir -p logs
21
+ sbatch "$0"
22
+ exit
23
+ fi
24
+
25
+ set -euo pipefail
26
+
27
+ # symlink logs/latest.out and logs/latest.err
28
+ ln -f -s $SLURM_JOB_ID.out logs/latest.out
29
+ ln -f -s $SLURM_JOB_ID.err logs/latest.err
30
+
31
+ KILL_SWITCH_PATH=kill-switch-$VARIANT
32
+ CHECKPOINT_PATH=checkpoints_$VARIANT
33
+ TENSORBOARD_PATH=tensorboard_$VARIANT
34
+
35
+ # Data
36
+ VOCAB_FILE="gpt2/vocab.json"
37
+ MERGE_FILE="gpt2/merges.txt"
38
+
39
+ TRAIN_DATA_PATH=train42b.txt
40
+ # "train: 1.0 0:1 /scratch/project_462000119/data/c4_subsampled/gpt2tok_c4_en_42B_text_document"
41
+ VALID_DATA_PATH=val.txt
42
+ # "validation: 1.0 0:1 /scratch/project_462000119/data/c4_validation/gpt2tok_c4validation_rerun_text_document"
43
+
44
+ PP_SIZE=1
45
+ TP_SIZE=2
46
+
47
+ MICRO_BATCH_SIZE=2
48
+ GRADIENT_ACCUMULATION_STEPS=1
49
+ WORLD_SIZE=$((SLURM_GPUS_ON_NODE*SLURM_JOB_NUM_NODES))
50
+ GLOBAL_BATCH_SIZE=$((MICRO_BATCH_SIZE*WORLD_SIZE*GRADIENT_ACCUMULATION_STEPS))
51
+
52
+ # Model parameters
53
+ source model_params.sh
54
+ MODEL_PARAM=("${PARAM_4516M[@]}")
55
+ NHIDDEN=${MODEL_PARAM[0]}
56
+ FFN_HIDDEN_SIZE=${MODEL_PARAM[1]}
57
+ KV_SIZE=${MODEL_PARAM[2]}
58
+ NHEADS=${MODEL_PARAM[3]}
59
+ NLAYERS=${MODEL_PARAM[4]}
60
+ SEQ_LEN=2048
61
+
62
+ echo "Model parameters: d_model $NHIDDEN ffw_size $FFN_HIDDEN_SIZE kv_size $KV_SIZE n_heads $NHEADS n_layers $NLAYERS"
63
+
64
+ SAVE_INTERVAL=10000
65
+
66
+ # Tokens: 84_000_000_000
67
+ # -> Samples: 41_015_625.0
68
+ TRAIN_SAMPLES=41_015_625
69
+
70
+ OPTIMIZER_ARGS=" \
71
+ --optimizer adam \
72
+ --adam-beta1 0.9 \
73
+ --adam-beta2 0.95 \
74
+ --adam-eps 1e-8 \
75
+ --lr 2e-4 \
76
+ --min-lr 2e-5 \
77
+ --lr-decay-style cosine \
78
+ --lr-decay-samples $TRAIN_SAMPLES \
79
+ --lr-warmup-samples 410_156 \
80
+ --clip-grad 1.0 \
81
+ --weight-decay 1e-1 \
82
+ "
83
+
84
+ GPT_ARGS=" \
85
+ --num-layers $NLAYERS \
86
+ --hidden-size $NHIDDEN \
87
+ --num-attention-heads $NHEADS \
88
+ --kv-channels $KV_SIZE \
89
+ --ffn-hidden-size $FFN_HIDDEN_SIZE \
90
+ --seq-length $SEQ_LEN \
91
+ --max-position-embeddings $SEQ_LEN \
92
+ --micro-batch-size $MICRO_BATCH_SIZE \
93
+ --global-batch-size $GLOBAL_BATCH_SIZE \
94
+ --train-samples $TRAIN_SAMPLES \
95
+ --vocab-file $VOCAB_FILE \
96
+ --merge-file $MERGE_FILE \
97
+ --clip-grad 1.0 \
98
+ --kill-switch-path $KILL_SWITCH_PATH \
99
+ --bf16 \
100
+ $OPTIMIZER_ARGS \
101
+ "
102
+
103
+ OUTPUT_ARGS=" \
104
+ --log-interval 10 \
105
+ --save-interval $SAVE_INTERVAL \
106
+ --eval-interval 1000 \
107
+ --eval-iters 100 \
108
+ --tensorboard-dir $TENSORBOARD_PATH \
109
+ --tensorboard-queue-size 5 \
110
+ --log-timers-to-tensorboard \
111
+ --log-batch-size-to-tensorboard \
112
+ --log-validation-ppl-to-tensorboard \
113
+ "
114
+
115
+ ZERO_STAGE=0
116
+
117
+ mkdir -p ds_configs
118
+ DS_CONFIG_PATH="ds_configs/$SLURM_JOB_ID.json"
119
+
120
+ cat <<EOF > $DS_CONFIG_PATH
121
+ {
122
+ "train_micro_batch_size_per_gpu": $MICRO_BATCH_SIZE,
123
+ "train_batch_size": $GLOBAL_BATCH_SIZE,
124
+ "gradient_clipping": 1.0,
125
+ "zero_optimization": {
126
+ "stage": $ZERO_STAGE
127
+ },
128
+ "bf16": {
129
+ "enabled": true
130
+ },
131
+ "steps_per_print": 2000,
132
+ "wall_clock_breakdown": false
133
+ }
134
+ EOF
135
+
136
+ DEEPSPEED_ARGS=" \
137
+ --deepspeed \
138
+ --deepspeed_config $DS_CONFIG_PATH \
139
+ --zero-stage $ZERO_STAGE \
140
+ "
141
+
142
+ CMD=" \
143
+ Megatron-DeepSpeed/pretrain_gpt.py \
144
+ --tensor-model-parallel-size $TP_SIZE \
145
+ --pipeline-model-parallel-size $PP_SIZE \
146
+ $GPT_ARGS \
147
+ $OUTPUT_ARGS \
148
+ --save $CHECKPOINT_PATH \
149
+ --load $CHECKPOINT_PATH \
150
+ --train-weighted-split-paths-path $TRAIN_DATA_PATH \
151
+ --valid-weighted-split-paths-path $VALID_DATA_PATH \
152
+ --data-impl mmap \
153
+ --seed 4 \
154
+ $DEEPSPEED_ARGS \
155
+ "
156
+
157
+ echo $CMD
158
+
159
+ echo "START $SLURM_JOBID: $(date)"
160
+
161
+ # bash launch_srun.sh $CMD
162
+ srun --label launch.sh $CMD
163
+
164
+ echo "END $SLURM_JOBID: $(date)"
sbatch_4b284b84bc4v2seed1.sh ADDED
@@ -0,0 +1,164 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #!/bin/bash
2
+ #SBATCH --exclude=nid007571,nid007112,nid006774,nid007502,nid007506,nid007507,nid005145,nid006692,nid007218,nid007123,nid006124,nid006123,nid007496,nid007237,nid006852,nid007206,nid006947,nid007212,nid006977,nid007222,nid005444,nid007219,nid007493,nid007221,nid005300,nid005619,nid006118,nid005203,nid006113,nid006481,nid007077,nid005208,nid005207,nid005879,nid005901
3
+ #SBATCH --nodes=32
4
+ #SBATCH --ntasks-per-node=1
5
+ #SBATCH --cpus-per-task=40
6
+ #SBATCH --mem=256G
7
+ #SBATCH -p standard-g
8
+ #SBATCH -t 48:00:00
9
+ #SBATCH --gpus-per-node=mi250:8
10
+ #SBATCH --exclusive=user
11
+ #SBATCH --hint=nomultithread
12
+ #SBATCH --account=project_462000119
13
+ #SBATCH -o logs/%j.out
14
+ #SBATCH -e logs/%j.err
15
+
16
+ VARIANT=4b284b84bc4v2seed1
17
+
18
+ # if run without sbatch, invoke here
19
+ if [ -z $SLURM_JOB_ID ]; then
20
+ mkdir -p logs
21
+ sbatch "$0"
22
+ exit
23
+ fi
24
+
25
+ set -euo pipefail
26
+
27
+ # symlink logs/latest.out and logs/latest.err
28
+ ln -f -s $SLURM_JOB_ID.out logs/latest.out
29
+ ln -f -s $SLURM_JOB_ID.err logs/latest.err
30
+
31
+ KILL_SWITCH_PATH=kill-switch-$VARIANT
32
+ CHECKPOINT_PATH=checkpoints_$VARIANT
33
+ TENSORBOARD_PATH=tensorboard_$VARIANT
34
+
35
+ # Data
36
+ VOCAB_FILE="gpt2/vocab.json"
37
+ MERGE_FILE="gpt2/merges.txt"
38
+
39
+ TRAIN_DATA_PATH=train84b.txt
40
+ # "train: 1.0 0:1 /scratch/project_462000119/data/c4_subsampled/gpt2tok_c4_en_84B_text_document"
41
+ VALID_DATA_PATH=val.txt
42
+ # "validation: 1.0 0:1 /scratch/project_462000119/data/c4_validation/gpt2tok_c4validation_rerun_text_document"
43
+
44
+ PP_SIZE=1
45
+ TP_SIZE=2
46
+
47
+ MICRO_BATCH_SIZE=2
48
+ GRADIENT_ACCUMULATION_STEPS=1
49
+ WORLD_SIZE=$((SLURM_GPUS_ON_NODE*SLURM_JOB_NUM_NODES))
50
+ GLOBAL_BATCH_SIZE=$((MICRO_BATCH_SIZE*WORLD_SIZE*GRADIENT_ACCUMULATION_STEPS))
51
+
52
+ # Model parameters
53
+ source model_params.sh
54
+ MODEL_PARAM=("${PARAM_4516M[@]}")
55
+ NHIDDEN=${MODEL_PARAM[0]}
56
+ FFN_HIDDEN_SIZE=${MODEL_PARAM[1]}
57
+ KV_SIZE=${MODEL_PARAM[2]}
58
+ NHEADS=${MODEL_PARAM[3]}
59
+ NLAYERS=${MODEL_PARAM[4]}
60
+ SEQ_LEN=2048
61
+
62
+ echo "Model parameters: d_model $NHIDDEN ffw_size $FFN_HIDDEN_SIZE kv_size $KV_SIZE n_heads $NHEADS n_layers $NLAYERS"
63
+
64
+ SAVE_INTERVAL=10000
65
+
66
+ # Tokens: 84_000_000_000
67
+ # -> Samples: 41_015_625.0
68
+ TRAIN_SAMPLES=41_015_625
69
+
70
+ OPTIMIZER_ARGS=" \
71
+ --optimizer adam \
72
+ --adam-beta1 0.9 \
73
+ --adam-beta2 0.95 \
74
+ --adam-eps 1e-8 \
75
+ --lr 2e-4 \
76
+ --min-lr 2e-5 \
77
+ --lr-decay-style cosine \
78
+ --lr-decay-samples $TRAIN_SAMPLES \
79
+ --lr-warmup-samples 410_156 \
80
+ --clip-grad 1.0 \
81
+ --weight-decay 1e-1 \
82
+ "
83
+
84
+ GPT_ARGS=" \
85
+ --num-layers $NLAYERS \
86
+ --hidden-size $NHIDDEN \
87
+ --num-attention-heads $NHEADS \
88
+ --kv-channels $KV_SIZE \
89
+ --ffn-hidden-size $FFN_HIDDEN_SIZE \
90
+ --seq-length $SEQ_LEN \
91
+ --max-position-embeddings $SEQ_LEN \
92
+ --micro-batch-size $MICRO_BATCH_SIZE \
93
+ --global-batch-size $GLOBAL_BATCH_SIZE \
94
+ --train-samples $TRAIN_SAMPLES \
95
+ --vocab-file $VOCAB_FILE \
96
+ --merge-file $MERGE_FILE \
97
+ --clip-grad 1.0 \
98
+ --kill-switch-path $KILL_SWITCH_PATH \
99
+ --bf16 \
100
+ $OPTIMIZER_ARGS \
101
+ "
102
+
103
+ OUTPUT_ARGS=" \
104
+ --log-interval 10 \
105
+ --save-interval $SAVE_INTERVAL \
106
+ --eval-interval 1000 \
107
+ --eval-iters 1 \
108
+ --tensorboard-dir $TENSORBOARD_PATH \
109
+ --tensorboard-queue-size 5 \
110
+ --log-timers-to-tensorboard \
111
+ --log-batch-size-to-tensorboard \
112
+ --log-validation-ppl-to-tensorboard \
113
+ "
114
+
115
+ ZERO_STAGE=0
116
+
117
+ mkdir -p ds_configs
118
+ DS_CONFIG_PATH="ds_configs/$SLURM_JOB_ID.json"
119
+
120
+ cat <<EOF > $DS_CONFIG_PATH
121
+ {
122
+ "train_micro_batch_size_per_gpu": $MICRO_BATCH_SIZE,
123
+ "train_batch_size": $GLOBAL_BATCH_SIZE,
124
+ "gradient_clipping": 1.0,
125
+ "zero_optimization": {
126
+ "stage": $ZERO_STAGE
127
+ },
128
+ "bf16": {
129
+ "enabled": true
130
+ },
131
+ "steps_per_print": 2000,
132
+ "wall_clock_breakdown": false
133
+ }
134
+ EOF
135
+
136
+ DEEPSPEED_ARGS=" \
137
+ --deepspeed \
138
+ --deepspeed_config $DS_CONFIG_PATH \
139
+ --zero-stage $ZERO_STAGE \
140
+ "
141
+
142
+ CMD=" \
143
+ Megatron-DeepSpeed/pretrain_gpt.py \
144
+ --tensor-model-parallel-size $TP_SIZE \
145
+ --pipeline-model-parallel-size $PP_SIZE \
146
+ $GPT_ARGS \
147
+ $OUTPUT_ARGS \
148
+ --save $CHECKPOINT_PATH \
149
+ --load $CHECKPOINT_PATH \
150
+ --train-weighted-split-paths-path $TRAIN_DATA_PATH \
151
+ --valid-weighted-split-paths-path $VALID_DATA_PATH \
152
+ --data-impl mmap \
153
+ --seed 1 \
154
+ $DEEPSPEED_ARGS \
155
+ "
156
+
157
+ echo $CMD
158
+
159
+ echo "START $SLURM_JOBID: $(date)"
160
+
161
+ # bash launch_srun.sh $CMD
162
+ srun --label launch.sh $CMD
163
+
164
+ echo "END $SLURM_JOBID: $(date)"
sbatch_4b284b84bc4v2seed2.sh ADDED
@@ -0,0 +1,164 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #!/bin/bash
2
+ #SBATCH --exclude=nid007571,nid007112,nid006774,nid007502,nid007506,nid007507,nid005145,nid006692,nid007218,nid007123,nid006124,nid006123,nid007496,nid007237,nid006852,nid007206,nid006947,nid007212,nid006977,nid007222,nid005444,nid007219,nid007493,nid007221,nid005300,nid005619,nid006118,nid005203,nid006113,nid006481,nid007077,nid005208,nid005207,nid005879,nid005901
3
+ #SBATCH --nodes=32
4
+ #SBATCH --ntasks-per-node=1
5
+ #SBATCH --cpus-per-task=40
6
+ #SBATCH --mem=256G
7
+ #SBATCH -p standard-g
8
+ #SBATCH -t 48:00:00
9
+ #SBATCH --gpus-per-node=mi250:8
10
+ #SBATCH --exclusive=user
11
+ #SBATCH --hint=nomultithread
12
+ #SBATCH --account=project_462000119
13
+ #SBATCH -o logs/%j.out
14
+ #SBATCH -e logs/%j.err
15
+
16
+ VARIANT=4b284b84bc4v2seed2
17
+
18
+ # if run without sbatch, invoke here
19
+ if [ -z $SLURM_JOB_ID ]; then
20
+ mkdir -p logs
21
+ sbatch "$0"
22
+ exit
23
+ fi
24
+
25
+ set -euo pipefail
26
+
27
+ # symlink logs/latest.out and logs/latest.err
28
+ ln -f -s $SLURM_JOB_ID.out logs/latest.out
29
+ ln -f -s $SLURM_JOB_ID.err logs/latest.err
30
+
31
+ KILL_SWITCH_PATH=kill-switch-$VARIANT
32
+ CHECKPOINT_PATH=checkpoints_$VARIANT
33
+ TENSORBOARD_PATH=tensorboard_$VARIANT
34
+
35
+ # Data
36
+ VOCAB_FILE="gpt2/vocab.json"
37
+ MERGE_FILE="gpt2/merges.txt"
38
+
39
+ TRAIN_DATA_PATH=train84b.txt
40
+ # "train: 1.0 0:1 /scratch/project_462000119/data/c4_subsampled/gpt2tok_c4_en_84B_text_document"
41
+ VALID_DATA_PATH=val.txt
42
+ # "validation: 1.0 0:1 /scratch/project_462000119/data/c4_validation/gpt2tok_c4validation_rerun_text_document"
43
+
44
+ PP_SIZE=1
45
+ TP_SIZE=2
46
+
47
+ MICRO_BATCH_SIZE=2
48
+ GRADIENT_ACCUMULATION_STEPS=1
49
+ WORLD_SIZE=$((SLURM_GPUS_ON_NODE*SLURM_JOB_NUM_NODES))
50
+ GLOBAL_BATCH_SIZE=$((MICRO_BATCH_SIZE*WORLD_SIZE*GRADIENT_ACCUMULATION_STEPS))
51
+
52
+ # Model parameters
53
+ source model_params.sh
54
+ MODEL_PARAM=("${PARAM_4516M[@]}")
55
+ NHIDDEN=${MODEL_PARAM[0]}
56
+ FFN_HIDDEN_SIZE=${MODEL_PARAM[1]}
57
+ KV_SIZE=${MODEL_PARAM[2]}
58
+ NHEADS=${MODEL_PARAM[3]}
59
+ NLAYERS=${MODEL_PARAM[4]}
60
+ SEQ_LEN=2048
61
+
62
+ echo "Model parameters: d_model $NHIDDEN ffw_size $FFN_HIDDEN_SIZE kv_size $KV_SIZE n_heads $NHEADS n_layers $NLAYERS"
63
+
64
+ SAVE_INTERVAL=10000
65
+
66
+ # Tokens: 84_000_000_000
67
+ # -> Samples: 41_015_625.0
68
+ TRAIN_SAMPLES=41_015_625
69
+
70
+ OPTIMIZER_ARGS=" \
71
+ --optimizer adam \
72
+ --adam-beta1 0.9 \
73
+ --adam-beta2 0.95 \
74
+ --adam-eps 1e-8 \
75
+ --lr 2e-4 \
76
+ --min-lr 2e-5 \
77
+ --lr-decay-style cosine \
78
+ --lr-decay-samples $TRAIN_SAMPLES \
79
+ --lr-warmup-samples 410_156 \
80
+ --clip-grad 1.0 \
81
+ --weight-decay 1e-1 \
82
+ "
83
+
84
+ GPT_ARGS=" \
85
+ --num-layers $NLAYERS \
86
+ --hidden-size $NHIDDEN \
87
+ --num-attention-heads $NHEADS \
88
+ --kv-channels $KV_SIZE \
89
+ --ffn-hidden-size $FFN_HIDDEN_SIZE \
90
+ --seq-length $SEQ_LEN \
91
+ --max-position-embeddings $SEQ_LEN \
92
+ --micro-batch-size $MICRO_BATCH_SIZE \
93
+ --global-batch-size $GLOBAL_BATCH_SIZE \
94
+ --train-samples $TRAIN_SAMPLES \
95
+ --vocab-file $VOCAB_FILE \
96
+ --merge-file $MERGE_FILE \
97
+ --clip-grad 1.0 \
98
+ --kill-switch-path $KILL_SWITCH_PATH \
99
+ --bf16 \
100
+ $OPTIMIZER_ARGS \
101
+ "
102
+
103
+ OUTPUT_ARGS=" \
104
+ --log-interval 10 \
105
+ --save-interval $SAVE_INTERVAL \
106
+ --eval-interval 1000 \
107
+ --eval-iters 1 \
108
+ --tensorboard-dir $TENSORBOARD_PATH \
109
+ --tensorboard-queue-size 5 \
110
+ --log-timers-to-tensorboard \
111
+ --log-batch-size-to-tensorboard \
112
+ --log-validation-ppl-to-tensorboard \
113
+ "
114
+
115
+ ZERO_STAGE=0
116
+
117
+ mkdir -p ds_configs
118
+ DS_CONFIG_PATH="ds_configs/$SLURM_JOB_ID.json"
119
+
120
+ cat <<EOF > $DS_CONFIG_PATH
121
+ {
122
+ "train_micro_batch_size_per_gpu": $MICRO_BATCH_SIZE,
123
+ "train_batch_size": $GLOBAL_BATCH_SIZE,
124
+ "gradient_clipping": 1.0,
125
+ "zero_optimization": {
126
+ "stage": $ZERO_STAGE
127
+ },
128
+ "bf16": {
129
+ "enabled": true
130
+ },
131
+ "steps_per_print": 2000,
132
+ "wall_clock_breakdown": false
133
+ }
134
+ EOF
135
+
136
+ DEEPSPEED_ARGS=" \
137
+ --deepspeed \
138
+ --deepspeed_config $DS_CONFIG_PATH \
139
+ --zero-stage $ZERO_STAGE \
140
+ "
141
+
142
+ CMD=" \
143
+ Megatron-DeepSpeed/pretrain_gpt.py \
144
+ --tensor-model-parallel-size $TP_SIZE \
145
+ --pipeline-model-parallel-size $PP_SIZE \
146
+ $GPT_ARGS \
147
+ $OUTPUT_ARGS \
148
+ --save $CHECKPOINT_PATH \
149
+ --load $CHECKPOINT_PATH \
150
+ --train-weighted-split-paths-path $TRAIN_DATA_PATH \
151
+ --valid-weighted-split-paths-path $VALID_DATA_PATH \
152
+ --data-impl mmap \
153
+ --seed 2 \
154
+ $DEEPSPEED_ARGS \
155
+ "
156
+
157
+ echo $CMD
158
+
159
+ echo "START $SLURM_JOBID: $(date)"
160
+
161
+ # bash launch_srun.sh $CMD
162
+ srun --label launch.sh $CMD
163
+
164
+ echo "END $SLURM_JOBID: $(date)"
sbatch_4b284b84bc4v2seed3.sh ADDED
@@ -0,0 +1,164 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #!/bin/bash
2
+ #SBATCH --exclude=nid007571,nid007112,nid006774,nid007502,nid007506,nid007507,nid005145,nid006692,nid007218,nid007123,nid006124,nid006123,nid007496,nid007237,nid006852,nid007206,nid006947,nid007212,nid006977,nid007222,nid005444,nid007219,nid007493,nid007221,nid005300,nid005619,nid006118,nid005203,nid006113,nid006481,nid007077,nid005208,nid005207,nid005879,nid005901
3
+ #SBATCH --nodes=32
4
+ #SBATCH --ntasks-per-node=1
5
+ #SBATCH --cpus-per-task=40
6
+ #SBATCH --mem=256G
7
+ #SBATCH -p standard-g
8
+ #SBATCH -t 48:00:00
9
+ #SBATCH --gpus-per-node=mi250:8
10
+ #SBATCH --exclusive=user
11
+ #SBATCH --hint=nomultithread
12
+ #SBATCH --account=project_462000119
13
+ #SBATCH -o logs/%j.out
14
+ #SBATCH -e logs/%j.err
15
+
16
+ VARIANT=4b284b84bc4v2seed3
17
+
18
+ # if run without sbatch, invoke here
19
+ if [ -z $SLURM_JOB_ID ]; then
20
+ mkdir -p logs
21
+ sbatch "$0"
22
+ exit
23
+ fi
24
+
25
+ set -euo pipefail
26
+
27
+ # symlink logs/latest.out and logs/latest.err
28
+ ln -f -s $SLURM_JOB_ID.out logs/latest.out
29
+ ln -f -s $SLURM_JOB_ID.err logs/latest.err
30
+
31
+ KILL_SWITCH_PATH=kill-switch-$VARIANT
32
+ CHECKPOINT_PATH=checkpoints_$VARIANT
33
+ TENSORBOARD_PATH=tensorboard_$VARIANT
34
+
35
+ # Data
36
+ VOCAB_FILE="gpt2/vocab.json"
37
+ MERGE_FILE="gpt2/merges.txt"
38
+
39
+ TRAIN_DATA_PATH=train84b.txt
40
+ # "train: 1.0 0:1 /scratch/project_462000119/data/c4_subsampled/gpt2tok_c4_en_84B_text_document"
41
+ VALID_DATA_PATH=val.txt
42
+ # "validation: 1.0 0:1 /scratch/project_462000119/data/c4_validation/gpt2tok_c4validation_rerun_text_document"
43
+
44
+ PP_SIZE=1
45
+ TP_SIZE=2
46
+
47
+ MICRO_BATCH_SIZE=2
48
+ GRADIENT_ACCUMULATION_STEPS=1
49
+ WORLD_SIZE=$((SLURM_GPUS_ON_NODE*SLURM_JOB_NUM_NODES))
50
+ GLOBAL_BATCH_SIZE=$((MICRO_BATCH_SIZE*WORLD_SIZE*GRADIENT_ACCUMULATION_STEPS))
51
+
52
+ # Model parameters
53
+ source model_params.sh
54
+ MODEL_PARAM=("${PARAM_4516M[@]}")
55
+ NHIDDEN=${MODEL_PARAM[0]}
56
+ FFN_HIDDEN_SIZE=${MODEL_PARAM[1]}
57
+ KV_SIZE=${MODEL_PARAM[2]}
58
+ NHEADS=${MODEL_PARAM[3]}
59
+ NLAYERS=${MODEL_PARAM[4]}
60
+ SEQ_LEN=2048
61
+
62
+ echo "Model parameters: d_model $NHIDDEN ffw_size $FFN_HIDDEN_SIZE kv_size $KV_SIZE n_heads $NHEADS n_layers $NLAYERS"
63
+
64
+ SAVE_INTERVAL=10000
65
+
66
+ # Tokens: 84_000_000_000
67
+ # -> Samples: 41_015_625.0
68
+ TRAIN_SAMPLES=41_015_625
69
+
70
+ OPTIMIZER_ARGS=" \
71
+ --optimizer adam \
72
+ --adam-beta1 0.9 \
73
+ --adam-beta2 0.95 \
74
+ --adam-eps 1e-8 \
75
+ --lr 2e-4 \
76
+ --min-lr 2e-5 \
77
+ --lr-decay-style cosine \
78
+ --lr-decay-samples $TRAIN_SAMPLES \
79
+ --lr-warmup-samples 410_156 \
80
+ --clip-grad 1.0 \
81
+ --weight-decay 1e-1 \
82
+ "
83
+
84
+ GPT_ARGS=" \
85
+ --num-layers $NLAYERS \
86
+ --hidden-size $NHIDDEN \
87
+ --num-attention-heads $NHEADS \
88
+ --kv-channels $KV_SIZE \
89
+ --ffn-hidden-size $FFN_HIDDEN_SIZE \
90
+ --seq-length $SEQ_LEN \
91
+ --max-position-embeddings $SEQ_LEN \
92
+ --micro-batch-size $MICRO_BATCH_SIZE \
93
+ --global-batch-size $GLOBAL_BATCH_SIZE \
94
+ --train-samples $TRAIN_SAMPLES \
95
+ --vocab-file $VOCAB_FILE \
96
+ --merge-file $MERGE_FILE \
97
+ --clip-grad 1.0 \
98
+ --kill-switch-path $KILL_SWITCH_PATH \
99
+ --bf16 \
100
+ $OPTIMIZER_ARGS \
101
+ "
102
+
103
+ OUTPUT_ARGS=" \
104
+ --log-interval 10 \
105
+ --save-interval $SAVE_INTERVAL \
106
+ --eval-interval 1000 \
107
+ --eval-iters 1 \
108
+ --tensorboard-dir $TENSORBOARD_PATH \
109
+ --tensorboard-queue-size 5 \
110
+ --log-timers-to-tensorboard \
111
+ --log-batch-size-to-tensorboard \
112
+ --log-validation-ppl-to-tensorboard \
113
+ "
114
+
115
+ ZERO_STAGE=0
116
+
117
+ mkdir -p ds_configs
118
+ DS_CONFIG_PATH="ds_configs/$SLURM_JOB_ID.json"
119
+
120
+ cat <<EOF > $DS_CONFIG_PATH
121
+ {
122
+ "train_micro_batch_size_per_gpu": $MICRO_BATCH_SIZE,
123
+ "train_batch_size": $GLOBAL_BATCH_SIZE,
124
+ "gradient_clipping": 1.0,
125
+ "zero_optimization": {
126
+ "stage": $ZERO_STAGE
127
+ },
128
+ "bf16": {
129
+ "enabled": true
130
+ },
131
+ "steps_per_print": 2000,
132
+ "wall_clock_breakdown": false
133
+ }
134
+ EOF
135
+
136
+ DEEPSPEED_ARGS=" \
137
+ --deepspeed \
138
+ --deepspeed_config $DS_CONFIG_PATH \
139
+ --zero-stage $ZERO_STAGE \
140
+ "
141
+
142
+ CMD=" \
143
+ Megatron-DeepSpeed/pretrain_gpt.py \
144
+ --tensor-model-parallel-size $TP_SIZE \
145
+ --pipeline-model-parallel-size $PP_SIZE \
146
+ $GPT_ARGS \
147
+ $OUTPUT_ARGS \
148
+ --save $CHECKPOINT_PATH \
149
+ --load $CHECKPOINT_PATH \
150
+ --train-weighted-split-paths-path $TRAIN_DATA_PATH \
151
+ --valid-weighted-split-paths-path $VALID_DATA_PATH \
152
+ --data-impl mmap \
153
+ --seed 3 \
154
+ $DEEPSPEED_ARGS \
155
+ "
156
+
157
+ echo $CMD
158
+
159
+ echo "START $SLURM_JOBID: $(date)"
160
+
161
+ # bash launch_srun.sh $CMD
162
+ srun --label launch.sh $CMD
163
+
164
+ echo "END $SLURM_JOBID: $(date)"
sbatch_4b284b84bc4v2seed4.sh ADDED
@@ -0,0 +1,164 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #!/bin/bash
2
+ #SBATCH --exclude=nid007571,nid007112,nid006774,nid007502,nid007506,nid007507,nid005145,nid006692,nid007218,nid007123,nid006124,nid006123,nid007496,nid007237,nid006852,nid007206,nid006947,nid007212,nid006977,nid007222,nid005444,nid007219,nid007493,nid007221,nid005300,nid005619,nid006118,nid005203,nid006113,nid006481,nid007077,nid005208,nid005207,nid005879,nid005901
3
+ #SBATCH --nodes=32
4
+ #SBATCH --ntasks-per-node=1
5
+ #SBATCH --cpus-per-task=40
6
+ #SBATCH --mem=256G
7
+ #SBATCH -p standard-g
8
+ #SBATCH -t 48:00:00
9
+ #SBATCH --gpus-per-node=mi250:8
10
+ #SBATCH --exclusive=user
11
+ #SBATCH --hint=nomultithread
12
+ #SBATCH --account=project_462000119
13
+ #SBATCH -o logs/%j.out
14
+ #SBATCH -e logs/%j.err
15
+
16
+ VARIANT=4b284b84bc4v2seed4
17
+
18
+ # if run without sbatch, invoke here
19
+ if [ -z $SLURM_JOB_ID ]; then
20
+ mkdir -p logs
21
+ sbatch "$0"
22
+ exit
23
+ fi
24
+
25
+ set -euo pipefail
26
+
27
+ # symlink logs/latest.out and logs/latest.err
28
+ ln -f -s $SLURM_JOB_ID.out logs/latest.out
29
+ ln -f -s $SLURM_JOB_ID.err logs/latest.err
30
+
31
+ KILL_SWITCH_PATH=kill-switch-$VARIANT
32
+ CHECKPOINT_PATH=checkpoints_$VARIANT
33
+ TENSORBOARD_PATH=tensorboard_$VARIANT
34
+
35
+ # Data
36
+ VOCAB_FILE="gpt2/vocab.json"
37
+ MERGE_FILE="gpt2/merges.txt"
38
+
39
+ TRAIN_DATA_PATH=train84b.txt
40
+ # "train: 1.0 0:1 /scratch/project_462000119/data/c4_subsampled/gpt2tok_c4_en_84B_text_document"
41
+ VALID_DATA_PATH=val.txt
42
+ # "validation: 1.0 0:1 /scratch/project_462000119/data/c4_validation/gpt2tok_c4validation_rerun_text_document"
43
+
44
+ PP_SIZE=1
45
+ TP_SIZE=2
46
+
47
+ MICRO_BATCH_SIZE=2
48
+ GRADIENT_ACCUMULATION_STEPS=1
49
+ WORLD_SIZE=$((SLURM_GPUS_ON_NODE*SLURM_JOB_NUM_NODES))
50
+ GLOBAL_BATCH_SIZE=$((MICRO_BATCH_SIZE*WORLD_SIZE*GRADIENT_ACCUMULATION_STEPS))
51
+
52
+ # Model parameters
53
+ source model_params.sh
54
+ MODEL_PARAM=("${PARAM_4516M[@]}")
55
+ NHIDDEN=${MODEL_PARAM[0]}
56
+ FFN_HIDDEN_SIZE=${MODEL_PARAM[1]}
57
+ KV_SIZE=${MODEL_PARAM[2]}
58
+ NHEADS=${MODEL_PARAM[3]}
59
+ NLAYERS=${MODEL_PARAM[4]}
60
+ SEQ_LEN=2048
61
+
62
+ echo "Model parameters: d_model $NHIDDEN ffw_size $FFN_HIDDEN_SIZE kv_size $KV_SIZE n_heads $NHEADS n_layers $NLAYERS"
63
+
64
+ SAVE_INTERVAL=10000
65
+
66
+ # Tokens: 84_000_000_000
67
+ # -> Samples: 41_015_625.0
68
+ TRAIN_SAMPLES=41_015_625
69
+
70
+ OPTIMIZER_ARGS=" \
71
+ --optimizer adam \
72
+ --adam-beta1 0.9 \
73
+ --adam-beta2 0.95 \
74
+ --adam-eps 1e-8 \
75
+ --lr 2e-4 \
76
+ --min-lr 2e-5 \
77
+ --lr-decay-style cosine \
78
+ --lr-decay-samples $TRAIN_SAMPLES \
79
+ --lr-warmup-samples 410_156 \
80
+ --clip-grad 1.0 \
81
+ --weight-decay 1e-1 \
82
+ "
83
+
84
+ GPT_ARGS=" \
85
+ --num-layers $NLAYERS \
86
+ --hidden-size $NHIDDEN \
87
+ --num-attention-heads $NHEADS \
88
+ --kv-channels $KV_SIZE \
89
+ --ffn-hidden-size $FFN_HIDDEN_SIZE \
90
+ --seq-length $SEQ_LEN \
91
+ --max-position-embeddings $SEQ_LEN \
92
+ --micro-batch-size $MICRO_BATCH_SIZE \
93
+ --global-batch-size $GLOBAL_BATCH_SIZE \
94
+ --train-samples $TRAIN_SAMPLES \
95
+ --vocab-file $VOCAB_FILE \
96
+ --merge-file $MERGE_FILE \
97
+ --clip-grad 1.0 \
98
+ --kill-switch-path $KILL_SWITCH_PATH \
99
+ --bf16 \
100
+ $OPTIMIZER_ARGS \
101
+ "
102
+
103
+ OUTPUT_ARGS=" \
104
+ --log-interval 10 \
105
+ --save-interval $SAVE_INTERVAL \
106
+ --eval-interval 1000 \
107
+ --eval-iters 1 \
108
+ --tensorboard-dir $TENSORBOARD_PATH \
109
+ --tensorboard-queue-size 5 \
110
+ --log-timers-to-tensorboard \
111
+ --log-batch-size-to-tensorboard \
112
+ --log-validation-ppl-to-tensorboard \
113
+ "
114
+
115
+ ZERO_STAGE=0
116
+
117
+ mkdir -p ds_configs
118
+ DS_CONFIG_PATH="ds_configs/$SLURM_JOB_ID.json"
119
+
120
+ cat <<EOF > $DS_CONFIG_PATH
121
+ {
122
+ "train_micro_batch_size_per_gpu": $MICRO_BATCH_SIZE,
123
+ "train_batch_size": $GLOBAL_BATCH_SIZE,
124
+ "gradient_clipping": 1.0,
125
+ "zero_optimization": {
126
+ "stage": $ZERO_STAGE
127
+ },
128
+ "bf16": {
129
+ "enabled": true
130
+ },
131
+ "steps_per_print": 2000,
132
+ "wall_clock_breakdown": false
133
+ }
134
+ EOF
135
+
136
+ DEEPSPEED_ARGS=" \
137
+ --deepspeed \
138
+ --deepspeed_config $DS_CONFIG_PATH \
139
+ --zero-stage $ZERO_STAGE \
140
+ "
141
+
142
+ CMD=" \
143
+ Megatron-DeepSpeed/pretrain_gpt.py \
144
+ --tensor-model-parallel-size $TP_SIZE \
145
+ --pipeline-model-parallel-size $PP_SIZE \
146
+ $GPT_ARGS \
147
+ $OUTPUT_ARGS \
148
+ --save $CHECKPOINT_PATH \
149
+ --load $CHECKPOINT_PATH \
150
+ --train-weighted-split-paths-path $TRAIN_DATA_PATH \
151
+ --valid-weighted-split-paths-path $VALID_DATA_PATH \
152
+ --data-impl mmap \
153
+ --seed 4 \
154
+ $DEEPSPEED_ARGS \
155
+ "
156
+
157
+ echo $CMD
158
+
159
+ echo "START $SLURM_JOBID: $(date)"
160
+
161
+ # bash launch_srun.sh $CMD
162
+ srun --label launch.sh $CMD
163
+
164
+ echo "END $SLURM_JOBID: $(date)"
tensorboard/tensorboard_4b284b12bc4seed1/events.out.tfevents.1676595687.nid006969.74604.0 ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:be8f3ee7320ed5480a7bd94308f8e4f2d6bee9adc9a8c1321878460f05343900
3
+ size 40
tensorboard/tensorboard_4b284b12bc4seed1/events.out.tfevents.1676596170.nid006963.99099.0 ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:c262b58569ce8bfe9d1f4813d5e6f5e80466376802f3e1868a019a99fe918bc2
3
+ size 40
tensorboard/tensorboard_4b284b12bc4seed1/events.out.tfevents.1676596625.nid006963.104051.0 ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:2559d5af4e49ae1b8f5c896684f1fbee0ba8b9767dca218a1884c09d09f9278a
3
+ size 40
tensorboard/tensorboard_4b284b12bc4seed1/events.out.tfevents.1676597074.nid006963.112349.0 ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:34080976df030176670adb62f21e1b9f92287eea8cb2eb7eada4ea04ef9bd756
3
+ size 40
tensorboard/tensorboard_4b284b12bc4seed1/events.out.tfevents.1676597574.nid005516.63089.0 ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:c62fed5ace459a55d74ac5f83c7e9b98d5ac4c282dade0858678f9a4d12a867f
3
+ size 40
tensorboard/tensorboard_4b284b12bc4seed1/events.out.tfevents.1676598062.nid005516.71283.0 ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:2ee80b2e112e41e1e2a15748c875a129f135495c575d34dc7ce5f3425ed4a916
3
+ size 40
tensorboard/tensorboard_4b284b12bc4seed1/events.out.tfevents.1676598563.nid005516.76684.0 ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:fa45a0c9c0caaf07d0c0e0cbc1a14963a847bcd0e6cc927fe4588041270e2384
3
+ size 40
tensorboard/tensorboard_4b284b12bc4seed1/events.out.tfevents.1676599066.nid005516.84582.0 ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:a10c35aa839d5fa503aa16b912d38e4e5830d9c31a7573deaf300e13854b4a5a
3
+ size 40
tensorboard/tensorboard_4b284b12bc4seed1/events.out.tfevents.1676599553.nid006969.122479.0 ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:9811fefbfcb2332e03bc33496060997394eff60052a3c6fb9354960271c36565
3
+ size 40
tensorboard/tensorboard_4b284b12bc4seed1/events.out.tfevents.1676600044.nid006969.127475.0 ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:7a4ef06a5b9671786234c21da4aff88500b10dc226ad065a964009010e1dfbb1
3
+ size 40
tensorboard/tensorboard_4b284b12bc4seed1/events.out.tfevents.1676600573.nid006969.5853.0 ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:98222165f4d32eafe1a90b9ca3fd98b2fd8a2ec91ec79dd84636a50625bd960c
3
+ size 40
tensorboard/tensorboard_4b284b12bc4seed1/events.out.tfevents.1677022444.nid006885.88040.0 ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:abe057e7f2bd7e1727d47572da3435c86b301cf3b60df08cc16411846fcd21ed
3
+ size 40
tensorboard/tensorboard_4b284b12bc4seed1/events.out.tfevents.1677022836.nid005287.20697.0 ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:95b175d9b26bf9c0e825afe990b7921a85e652afb99b6077f47241c2e2722782
3
+ size 40
tensorboard/tensorboard_4b284b12bc4seed1/events.out.tfevents.1677023221.nid005197.112586.0 ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:573ec1d0ed80288d4b1c78f3fbc6c1b51c3dbb3592b2ecaedf2638b0ceadd25f
3
+ size 40
tensorboard/tensorboard_4b284b12bc4seed1/events.out.tfevents.1677023622.nid005332.86111.0 ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:ff1ad01f4fdbfdb80f95811b5fd70b636889b46bcf32dca5b7d524078789b934
3
+ size 40
tensorboard/tensorboard_4b284b12bc4seed1/events.out.tfevents.1677024036.nid006101.78835.0 ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:6b2f92ac49b4fe0f0274332e98e88c0e6106b6642af6b90b0d3fe60ca5fa5f19
3
+ size 40
tensorboard/tensorboard_4b284b12bc4seed1/events.out.tfevents.1677024440.nid005197.4683.0 ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:7df39ec6a86b46ce9c338d03b5362fc736cd2bc8315002810d7667121bc01fbe
3
+ size 40
tensorboard/tensorboard_4b284b12bc4seed1/events.out.tfevents.1677024845.nid005332.99971.0 ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:d2eae861d8bc44fd7612e7cc231b5c246de7f7c273243d22ba46e18e32c779fe
3
+ size 40
tensorboard/tensorboard_4b284b12bc4seed1/events.out.tfevents.1677025248.nid006885.2652.0 ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:88981eb1d544cab34d179d418d22be2dec2eece32c82723e697c56960b3c136e
3
+ size 40
tensorboard/tensorboard_4b284b12bc4seed1/events.out.tfevents.1677025666.nid006444.5012.0 ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:2086c1f1fb1108058a65bf1d535ab38fabd3c7aa7ff5920749724b2e5efd2565
3
+ size 40
tensorboard/tensorboard_4b284b12bc4seed1/events.out.tfevents.1677026072.nid006885.11385.0 ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:f05751cb341b427d533103be7c1903fc0a9af1365fc62e593ced6c89cf645fff
3
+ size 40
tensorboard/tensorboard_4b284b12bc4seed1/events.out.tfevents.1677026470.nid006444.14023.0 ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:f3ce4570c710f180f5fb4cf56be9378684fa850b98ac363a4b27f8cb34c0a5f1
3
+ size 40
tensorboard/tensorboard_4b284b12bc4seed1/events.out.tfevents.1677026881.nid006101.114596.0 ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:a61d8d0cd32d8b06dc2a2983c60d18d2d3e08a0004bf64db4566a3e0689f46ed
3
+ size 40
tensorboard/tensorboard_4b284b12bc4seed1/events.out.tfevents.1677027297.nid005788.121184.0 ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:fdf79eeb7dc69193aa7b3de5ef73ebf958804cd5166631674d9aed21b6e2c154
3
+ size 40
tensorboard/tensorboard_4b284b12bc4seed1/events.out.tfevents.1677027709.nid007155.46377.0 ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:3a26e14478388f6bc625a7960e782d6348f48f95fec318f9cb9a813373e5b996
3
+ size 40
tensorboard/tensorboard_4b284b12bc4seed1/events.out.tfevents.1677028112.nid007191.46020.0 ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:108dd9584b83ec991001547b9705e7a902c65a337f82bc9a145ebfe0711e5f9e
3
+ size 40