remove illegal combo bench

llama-1B/16_GPUS/dp-1_tp-16_pp-1_mbz-1024/bench.slurm

@@ -1,111 +0,0 @@
-#!/bin/bash
-
-#SBATCH --job-name=bench_cluster
-#SBATCH --time=00:59:00
-#SBATCH --partition=hopper-prod
-#SBATCH --nodes=2
-#SBATCH --gres=gpu:8
-#SBATCH --qos=high
-#SBATCH --ntasks-per-node=1
-#SBATCH --cpus-per-task=96
-#SBATCH --exclusive
-#SBATCH --output=/fsx/ferdinandmom/ferdinand-hf/bench_cluster/results/llama-1B/16_GPUS/dp-1_tp-16_pp-1_mbz-1024/log.out
-#SBATCH --error=/fsx/ferdinandmom/ferdinand-hf/bench_cluster/results/llama-1B/16_GPUS/dp-1_tp-16_pp-1_mbz-1024/log.out
-
-# Function to update status based on squeue output
-update_status() {
-    job_id=$1
-    status_file=$2
-    # For unknown reasons, it doenst update status for pending. It only works for running 
-    while true; do
-        job_status=$(squeue --job $job_id --noheader --format=%T)
-        echo "Job status: $job_status"
-        if [ -z "$job_status" ]; then
-            # Job has finished or is not found
-            break
-        elif [ "$job_status" = "RUNNING" ]; then
-            printf "running" > $status_file
-            break
-        fi
-        sleep 10
-    done
-}
-
-# Misc initializations.
-echo "========================"
-echo "START TIME: $(date)"
-source /fsx/ferdinandmom/miniforge3/etc/profile.d/conda.sh
-conda activate /fsx/ferdinandmom/miniforge3/envs/env-bench-cluster
-echo python3 version = $(python3 --version)
-echo "========================"
-
-# Slurm stuff
-export HOSTNAMES=$(scontrol show hostnames "$SLURM_JOB_NODELIST")
-export MASTER_ADDR=$(scontrol show hostnames "$SLURM_JOB_NODELIST" | head -n 1)
-export MASTER_PORT=$((1024 + RANDOM % 64511))
-
-export TMPDIR=/scratch
-export HF_DATASETS_CACHE="/admin/home/ferdinand_mom/.cache"
-export CUBLAS_WORKSPACE_CONFIG=":4096:8"
-export CUDA_DEVICE_MAX_CONNECTIONS="1"
-
-huggingface-cli login --token $HUGGINGFACE_TOKEN
-
-
-NANOTRON_REPO="/fsx/ferdinandmom/ferdinand-hf/bench_cluster/nanotron"
-CMD="$NANOTRON_REPO/run_train.py --config-file /fsx/ferdinandmom/ferdinand-hf/bench_cluster/results/llama-1B/16_GPUS/dp-1_tp-16_pp-1_mbz-1024/config.yaml"
-
-LAUNCHER="torchrun \
-   --nproc_per_node 8 \
-   --nnodes 2 \
-   --rdzv_endpoint ${MASTER_ADDR}:${MASTER_PORT} \
-   --rdzv_backend c10d \
-   --max_restarts 0 \
-   --tee 3 \
-   --node_rank ${SLURM_PROCID}"
-
-# Checkout the bench_cluster branch
-cd $NANOTRON_REPO
-git checkout bench_cluster
-cd ..
-# Get the current job ID
-job_id=${SLURM_JOB_ID}
-
-# Update status to "pending" or "running" in the background
-update_status $job_id /fsx/ferdinandmom/ferdinand-hf/bench_cluster/results/llama-1B/16_GPUS/dp-1_tp-16_pp-1_mbz-1024/status.txt &
-
-# Run the main command
-srun -u $LAUNCHER $CMD
-exit_status=$?
-
-# Update status based on the exit status of `srun`
-if [ $exit_status -eq 0 ]; then
-    printf "completed" > /fsx/ferdinandmom/ferdinand-hf/bench_cluster/results/llama-1B/16_GPUS/dp-1_tp-16_pp-1_mbz-1024/status.txt
-else
-    if grep -q "OutOfMemoryError" /fsx/ferdinandmom/ferdinand-hf/bench_cluster/results/llama-1B/16_GPUS/dp-1_tp-16_pp-1_mbz-1024/log.out; then
-        printf "oom" > /fsx/ferdinandmom/ferdinand-hf/bench_cluster/results/llama-1B/16_GPUS/dp-1_tp-16_pp-1_mbz-1024/status.txt
-    elif grep -q " CUDA error: an illegal memory access" /fsx/ferdinandmom/ferdinand-hf/bench_cluster/results/llama-1B/16_GPUS/dp-1_tp-16_pp-1_mbz-1024/log.out; then
-        printf "oom" > /fsx/ferdinandmom/ferdinand-hf/bench_cluster/results/llama-1B/16_GPUS/dp-1_tp-16_pp-1_mbz-1024/status.txt
-    elif grep -q "Timeout at NCCL" /fsx/ferdinandmom/ferdinand-hf/bench_cluster/results/llama-1B/16_GPUS/dp-1_tp-16_pp-1_mbz-1024/log.out; then
-        printf "timeout" > /fsx/ferdinandmom/ferdinand-hf/bench_cluster/results/llama-1B/16_GPUS/dp-1_tp-16_pp-1_mbz-1024/status.txt
-    else
-        printf "fail" > /fsx/ferdinandmom/ferdinand-hf/bench_cluster/results/llama-1B/16_GPUS/dp-1_tp-16_pp-1_mbz-1024/status.txt
-    fi 
-fi
-
-# Run the report script if the job completed successfully
-if [ $exit_status -eq 0 ]; then
-    python /fsx/ferdinandmom/ferdinand-hf/bench_cluster/main.py report --inp_dir /fsx/ferdinandmom/ferdinand-hf/bench_cluster/results/llama-1B/16_GPUS/dp-1_tp-16_pp-1_mbz-1024 --is_logs
-    python /fsx/ferdinandmom/ferdinand-hf/bench_cluster/main.py report --inp_dir /fsx/ferdinandmom/ferdinand-hf/bench_cluster/results/llama-1B/16_GPUS/dp-1_tp-16_pp-1_mbz-1024 --is_profiler
-fi
-
-
-# Push to hub the folder using huggingface_cli
-huggingface-cli upload nanotron/bench_cluster /fsx/ferdinandmom/ferdinand-hf/bench_cluster/results/llama-1B/16_GPUS/dp-1_tp-16_pp-1_mbz-1024 llama-1B/16_GPUS/dp-1_tp-16_pp-1_mbz-1024 --commit-message "Upload llama-1B/16_GPUS/dp-1_tp-16_pp-1_mbz-1024"
-
-# Verify the upload
-if [ $? -eq 0 ]; then
-    echo "Uploading to Huggingface Hub successful"
-else
-    echo "Failed to upload to Huggingface Hub"
-fi

llama-1B/16_GPUS/dp-1_tp-16_pp-1_mbz-1024/config.yaml

@@ -1,90 +0,0 @@
-general:
-  project: bench_cluster
-  seed: 42
-model:
-  ddp_bucket_cap_mb: 25
-  dtype: bfloat16
-  init_method:
-    std: 0.025
-  make_vocab_size_divisible_by: 1
-  model_config:
-    bos_token_id: 1
-    eos_token_id: 2
-    hidden_act: silu
-    hidden_size: 2048
-    initializer_range: 0.02
-    intermediate_size: 4096
-    is_llama_config: true
-    max_position_embeddings: 4096
-    num_attention_heads: 32
-    num_hidden_layers: 24
-    num_key_value_heads: 32
-    pad_token_id: null
-    pretraining_tp: 1
-    rms_norm_eps: 1.0e-05
-    rope_scaling: null
-    rope_theta: 10000.0
-    tie_word_embeddings: true
-    use_cache: true
-    vocab_size: 50257
-optimizer:
-  accumulate_grad_in_fp32: true
-  clip_grad: 1.0
-  learning_rate_scheduler:
-    learning_rate: 0.0001
-    lr_decay_style: linear
-    lr_warmup_style: linear
-    lr_warmup_steps: 1
-    min_decay_lr: 1.0e-05
-  optimizer_factory:
-    adam_beta1: 0.9
-    adam_beta2: 0.95
-    adam_eps: 1.0e-08
-    name: adamW
-    torch_adam_is_fused: true
-  weight_decay: 0.01
-  zero_stage: 1
-parallelism:
-  dp: 1
-  expert_parallel_size: 1
-  pp: 1
-  pp_engine: 1f1b
-  tp: 16
-  tp_linear_async_communication: false
-  tp_mode: REDUCE_SCATTER
-profiler:
-  profiler_export_path: /fsx/ferdinandmom/ferdinand-hf/bench_cluster/results/llama-1B/16_GPUS/dp-1_tp-16_pp-1_mbz-1024
-tokenizer:
-  tokenizer_max_length: null
-  tokenizer_name_or_path: openai-community/gpt2
-  tokenizer_revision: null
-data_stages:
-- name: Training Stage
-  start_training_step: 1
-  data:
-    dataset:
-      dataset_overwrite_cache: false
-      dataset_processing_num_proc_per_process: 64
-      hf_dataset_config_name: null
-      hf_dataset_or_datasets: roneneldan/TinyStories
-      hf_dataset_splits: train
-      text_column_name: text
-    num_loading_workers: 32
-    seed: 42
-lighteval: null
-tokens:
-  train_steps: 20
-  val_check_interval: -1
-  batch_accumulation_per_replica: 1
-  limit_test_batches: 0
-  limit_val_batches: 0
-  micro_batch_size: 1024
-  sequence_length: 4096
-logging:
-  iteration_step_info_interval: 1
-  log_level: info
-  log_level_replica: info
-checkpoints:
-  checkpoint_interval: 100000
-  checkpoints_path: /dev/null
-  resume_checkpoint_path: null

llama-1B/16_GPUS/dp-1_tp-16_pp-1_mbz-1024/status.txt

@@ -1 +0,0 @@
-oom

llama-1B/16_GPUS/dp-1_tp-16_pp-1_mbz-128/bench.slurm

@@ -1,111 +0,0 @@
-#!/bin/bash
-
-#SBATCH --job-name=bench_cluster
-#SBATCH --time=00:59:00
-#SBATCH --partition=hopper-prod
-#SBATCH --nodes=2
-#SBATCH --gres=gpu:8
-#SBATCH --qos=high
-#SBATCH --ntasks-per-node=1
-#SBATCH --cpus-per-task=96
-#SBATCH --exclusive
-#SBATCH --output=/fsx/ferdinandmom/ferdinand-hf/bench_cluster/results/llama-1B/16_GPUS/dp-1_tp-16_pp-1_mbz-128/log.out
-#SBATCH --error=/fsx/ferdinandmom/ferdinand-hf/bench_cluster/results/llama-1B/16_GPUS/dp-1_tp-16_pp-1_mbz-128/log.out
-
-# Function to update status based on squeue output
-update_status() {
-    job_id=$1
-    status_file=$2
-    # For unknown reasons, it doenst update status for pending. It only works for running 
-    while true; do
-        job_status=$(squeue --job $job_id --noheader --format=%T)
-        echo "Job status: $job_status"
-        if [ -z "$job_status" ]; then
-            # Job has finished or is not found
-            break
-        elif [ "$job_status" = "RUNNING" ]; then
-            printf "running" > $status_file
-            break
-        fi
-        sleep 10
-    done
-}
-
-# Misc initializations.
-echo "========================"
-echo "START TIME: $(date)"
-source /fsx/ferdinandmom/miniforge3/etc/profile.d/conda.sh
-conda activate /fsx/ferdinandmom/miniforge3/envs/env-bench-cluster
-echo python3 version = $(python3 --version)
-echo "========================"
-
-# Slurm stuff
-export HOSTNAMES=$(scontrol show hostnames "$SLURM_JOB_NODELIST")
-export MASTER_ADDR=$(scontrol show hostnames "$SLURM_JOB_NODELIST" | head -n 1)
-export MASTER_PORT=$((1024 + RANDOM % 64511))
-
-export TMPDIR=/scratch
-export HF_DATASETS_CACHE="/admin/home/ferdinand_mom/.cache"
-export CUBLAS_WORKSPACE_CONFIG=":4096:8"
-export CUDA_DEVICE_MAX_CONNECTIONS="1"
-
-huggingface-cli login --token $HUGGINGFACE_TOKEN
-
-
-NANOTRON_REPO="/fsx/ferdinandmom/ferdinand-hf/bench_cluster/nanotron"
-CMD="$NANOTRON_REPO/run_train.py --config-file /fsx/ferdinandmom/ferdinand-hf/bench_cluster/results/llama-1B/16_GPUS/dp-1_tp-16_pp-1_mbz-128/config.yaml"
-
-LAUNCHER="torchrun \
-   --nproc_per_node 8 \
-   --nnodes 2 \
-   --rdzv_endpoint ${MASTER_ADDR}:${MASTER_PORT} \
-   --rdzv_backend c10d \
-   --max_restarts 0 \
-   --tee 3 \
-   --node_rank ${SLURM_PROCID}"
-
-# Checkout the bench_cluster branch
-cd $NANOTRON_REPO
-git checkout bench_cluster
-cd ..
-# Get the current job ID
-job_id=${SLURM_JOB_ID}
-
-# Update status to "pending" or "running" in the background
-update_status $job_id /fsx/ferdinandmom/ferdinand-hf/bench_cluster/results/llama-1B/16_GPUS/dp-1_tp-16_pp-1_mbz-128/status.txt &
-
-# Run the main command
-srun -u $LAUNCHER $CMD
-exit_status=$?
-
-# Update status based on the exit status of `srun`
-if [ $exit_status -eq 0 ]; then
-    printf "completed" > /fsx/ferdinandmom/ferdinand-hf/bench_cluster/results/llama-1B/16_GPUS/dp-1_tp-16_pp-1_mbz-128/status.txt
-else
-    if grep -q "OutOfMemoryError" /fsx/ferdinandmom/ferdinand-hf/bench_cluster/results/llama-1B/16_GPUS/dp-1_tp-16_pp-1_mbz-128/log.out; then
-        printf "oom" > /fsx/ferdinandmom/ferdinand-hf/bench_cluster/results/llama-1B/16_GPUS/dp-1_tp-16_pp-1_mbz-128/status.txt
-    elif grep -q " CUDA error: an illegal memory access" /fsx/ferdinandmom/ferdinand-hf/bench_cluster/results/llama-1B/16_GPUS/dp-1_tp-16_pp-1_mbz-128/log.out; then
-        printf "oom" > /fsx/ferdinandmom/ferdinand-hf/bench_cluster/results/llama-1B/16_GPUS/dp-1_tp-16_pp-1_mbz-128/status.txt
-    elif grep -q "Timeout at NCCL" /fsx/ferdinandmom/ferdinand-hf/bench_cluster/results/llama-1B/16_GPUS/dp-1_tp-16_pp-1_mbz-128/log.out; then
-        printf "timeout" > /fsx/ferdinandmom/ferdinand-hf/bench_cluster/results/llama-1B/16_GPUS/dp-1_tp-16_pp-1_mbz-128/status.txt
-    else
-        printf "fail" > /fsx/ferdinandmom/ferdinand-hf/bench_cluster/results/llama-1B/16_GPUS/dp-1_tp-16_pp-1_mbz-128/status.txt
-    fi 
-fi
-
-# Run the report script if the job completed successfully
-if [ $exit_status -eq 0 ]; then
-    python /fsx/ferdinandmom/ferdinand-hf/bench_cluster/main.py report --inp_dir /fsx/ferdinandmom/ferdinand-hf/bench_cluster/results/llama-1B/16_GPUS/dp-1_tp-16_pp-1_mbz-128 --is_logs
-    python /fsx/ferdinandmom/ferdinand-hf/bench_cluster/main.py report --inp_dir /fsx/ferdinandmom/ferdinand-hf/bench_cluster/results/llama-1B/16_GPUS/dp-1_tp-16_pp-1_mbz-128 --is_profiler
-fi
-
-
-# Push to hub the folder using huggingface_cli
-huggingface-cli upload nanotron/bench_cluster /fsx/ferdinandmom/ferdinand-hf/bench_cluster/results/llama-1B/16_GPUS/dp-1_tp-16_pp-1_mbz-128 llama-1B/16_GPUS/dp-1_tp-16_pp-1_mbz-128 --commit-message "Upload llama-1B/16_GPUS/dp-1_tp-16_pp-1_mbz-128"
-
-# Verify the upload
-if [ $? -eq 0 ]; then
-    echo "Uploading to Huggingface Hub successful"
-else
-    echo "Failed to upload to Huggingface Hub"
-fi

llama-1B/16_GPUS/dp-1_tp-16_pp-1_mbz-128/config.yaml

@@ -1,90 +0,0 @@
-general:
-  project: bench_cluster
-  seed: 42
-model:
-  ddp_bucket_cap_mb: 25
-  dtype: bfloat16
-  init_method:
-    std: 0.025
-  make_vocab_size_divisible_by: 1
-  model_config:
-    bos_token_id: 1
-    eos_token_id: 2
-    hidden_act: silu
-    hidden_size: 2048
-    initializer_range: 0.02
-    intermediate_size: 4096
-    is_llama_config: true
-    max_position_embeddings: 4096
-    num_attention_heads: 32
-    num_hidden_layers: 24
-    num_key_value_heads: 32
-    pad_token_id: null
-    pretraining_tp: 1
-    rms_norm_eps: 1.0e-05
-    rope_scaling: null
-    rope_theta: 10000.0
-    tie_word_embeddings: true
-    use_cache: true
-    vocab_size: 50257
-optimizer:
-  accumulate_grad_in_fp32: true
-  clip_grad: 1.0
-  learning_rate_scheduler:
-    learning_rate: 0.0001
-    lr_decay_style: linear
-    lr_warmup_style: linear
-    lr_warmup_steps: 1
-    min_decay_lr: 1.0e-05
-  optimizer_factory:
-    adam_beta1: 0.9
-    adam_beta2: 0.95
-    adam_eps: 1.0e-08
-    name: adamW
-    torch_adam_is_fused: true
-  weight_decay: 0.01
-  zero_stage: 1
-parallelism:
-  dp: 1
-  expert_parallel_size: 1
-  pp: 1
-  pp_engine: 1f1b
-  tp: 16
-  tp_linear_async_communication: false
-  tp_mode: REDUCE_SCATTER
-profiler:
-  profiler_export_path: /fsx/ferdinandmom/ferdinand-hf/bench_cluster/results/llama-1B/16_GPUS/dp-1_tp-16_pp-1_mbz-128
-tokenizer:
-  tokenizer_max_length: null
-  tokenizer_name_or_path: openai-community/gpt2
-  tokenizer_revision: null
-data_stages:
-- name: Training Stage
-  start_training_step: 1
-  data:
-    dataset:
-      dataset_overwrite_cache: false
-      dataset_processing_num_proc_per_process: 64
-      hf_dataset_config_name: null
-      hf_dataset_or_datasets: roneneldan/TinyStories
-      hf_dataset_splits: train
-      text_column_name: text
-    num_loading_workers: 32
-    seed: 42
-lighteval: null
-tokens:
-  train_steps: 20
-  val_check_interval: -1
-  batch_accumulation_per_replica: 8
-  limit_test_batches: 0
-  limit_val_batches: 0
-  micro_batch_size: 128
-  sequence_length: 4096
-logging:
-  iteration_step_info_interval: 1
-  log_level: info
-  log_level_replica: info
-checkpoints:
-  checkpoint_interval: 100000
-  checkpoints_path: /dev/null
-  resume_checkpoint_path: null

llama-1B/16_GPUS/dp-1_tp-16_pp-1_mbz-128/status.txt

@@ -1 +0,0 @@
-oom

llama-1B/16_GPUS/dp-1_tp-16_pp-1_mbz-256/bench.slurm

@@ -1,111 +0,0 @@
-#!/bin/bash
-
-#SBATCH --job-name=bench_cluster
-#SBATCH --time=00:59:00
-#SBATCH --partition=hopper-prod
-#SBATCH --nodes=2
-#SBATCH --gres=gpu:8
-#SBATCH --qos=high
-#SBATCH --ntasks-per-node=1
-#SBATCH --cpus-per-task=96
-#SBATCH --exclusive
-#SBATCH --output=/fsx/ferdinandmom/ferdinand-hf/bench_cluster/results/llama-1B/16_GPUS/dp-1_tp-16_pp-1_mbz-256/log.out
-#SBATCH --error=/fsx/ferdinandmom/ferdinand-hf/bench_cluster/results/llama-1B/16_GPUS/dp-1_tp-16_pp-1_mbz-256/log.out
-
-# Function to update status based on squeue output
-update_status() {
-    job_id=$1
-    status_file=$2
-    # For unknown reasons, it doenst update status for pending. It only works for running 
-    while true; do
-        job_status=$(squeue --job $job_id --noheader --format=%T)
-        echo "Job status: $job_status"
-        if [ -z "$job_status" ]; then
-            # Job has finished or is not found
-            break
-        elif [ "$job_status" = "RUNNING" ]; then
-            printf "running" > $status_file
-            break
-        fi
-        sleep 10
-    done
-}
-
-# Misc initializations.
-echo "========================"
-echo "START TIME: $(date)"
-source /fsx/ferdinandmom/miniforge3/etc/profile.d/conda.sh
-conda activate /fsx/ferdinandmom/miniforge3/envs/env-bench-cluster
-echo python3 version = $(python3 --version)
-echo "========================"
-
-# Slurm stuff
-export HOSTNAMES=$(scontrol show hostnames "$SLURM_JOB_NODELIST")
-export MASTER_ADDR=$(scontrol show hostnames "$SLURM_JOB_NODELIST" | head -n 1)
-export MASTER_PORT=$((1024 + RANDOM % 64511))
-
-export TMPDIR=/scratch
-export HF_DATASETS_CACHE="/admin/home/ferdinand_mom/.cache"
-export CUBLAS_WORKSPACE_CONFIG=":4096:8"
-export CUDA_DEVICE_MAX_CONNECTIONS="1"
-
-huggingface-cli login --token $HUGGINGFACE_TOKEN
-
-
-NANOTRON_REPO="/fsx/ferdinandmom/ferdinand-hf/bench_cluster/nanotron"
-CMD="$NANOTRON_REPO/run_train.py --config-file /fsx/ferdinandmom/ferdinand-hf/bench_cluster/results/llama-1B/16_GPUS/dp-1_tp-16_pp-1_mbz-256/config.yaml"
-
-LAUNCHER="torchrun \
-   --nproc_per_node 8 \
-   --nnodes 2 \
-   --rdzv_endpoint ${MASTER_ADDR}:${MASTER_PORT} \
-   --rdzv_backend c10d \
-   --max_restarts 0 \
-   --tee 3 \
-   --node_rank ${SLURM_PROCID}"
-
-# Checkout the bench_cluster branch
-cd $NANOTRON_REPO
-git checkout bench_cluster
-cd ..
-# Get the current job ID
-job_id=${SLURM_JOB_ID}
-
-# Update status to "pending" or "running" in the background
-update_status $job_id /fsx/ferdinandmom/ferdinand-hf/bench_cluster/results/llama-1B/16_GPUS/dp-1_tp-16_pp-1_mbz-256/status.txt &
-
-# Run the main command
-srun -u $LAUNCHER $CMD
-exit_status=$?
-
-# Update status based on the exit status of `srun`
-if [ $exit_status -eq 0 ]; then
-    printf "completed" > /fsx/ferdinandmom/ferdinand-hf/bench_cluster/results/llama-1B/16_GPUS/dp-1_tp-16_pp-1_mbz-256/status.txt
-else
-    if grep -q "OutOfMemoryError" /fsx/ferdinandmom/ferdinand-hf/bench_cluster/results/llama-1B/16_GPUS/dp-1_tp-16_pp-1_mbz-256/log.out; then
-        printf "oom" > /fsx/ferdinandmom/ferdinand-hf/bench_cluster/results/llama-1B/16_GPUS/dp-1_tp-16_pp-1_mbz-256/status.txt
-    elif grep -q " CUDA error: an illegal memory access" /fsx/ferdinandmom/ferdinand-hf/bench_cluster/results/llama-1B/16_GPUS/dp-1_tp-16_pp-1_mbz-256/log.out; then
-        printf "oom" > /fsx/ferdinandmom/ferdinand-hf/bench_cluster/results/llama-1B/16_GPUS/dp-1_tp-16_pp-1_mbz-256/status.txt
-    elif grep -q "Timeout at NCCL" /fsx/ferdinandmom/ferdinand-hf/bench_cluster/results/llama-1B/16_GPUS/dp-1_tp-16_pp-1_mbz-256/log.out; then
-        printf "timeout" > /fsx/ferdinandmom/ferdinand-hf/bench_cluster/results/llama-1B/16_GPUS/dp-1_tp-16_pp-1_mbz-256/status.txt
-    else
-        printf "fail" > /fsx/ferdinandmom/ferdinand-hf/bench_cluster/results/llama-1B/16_GPUS/dp-1_tp-16_pp-1_mbz-256/status.txt
-    fi 
-fi
-
-# Run the report script if the job completed successfully
-if [ $exit_status -eq 0 ]; then
-    python /fsx/ferdinandmom/ferdinand-hf/bench_cluster/main.py report --inp_dir /fsx/ferdinandmom/ferdinand-hf/bench_cluster/results/llama-1B/16_GPUS/dp-1_tp-16_pp-1_mbz-256 --is_logs
-    python /fsx/ferdinandmom/ferdinand-hf/bench_cluster/main.py report --inp_dir /fsx/ferdinandmom/ferdinand-hf/bench_cluster/results/llama-1B/16_GPUS/dp-1_tp-16_pp-1_mbz-256 --is_profiler
-fi
-
-
-# Push to hub the folder using huggingface_cli
-huggingface-cli upload nanotron/bench_cluster /fsx/ferdinandmom/ferdinand-hf/bench_cluster/results/llama-1B/16_GPUS/dp-1_tp-16_pp-1_mbz-256 llama-1B/16_GPUS/dp-1_tp-16_pp-1_mbz-256 --commit-message "Upload llama-1B/16_GPUS/dp-1_tp-16_pp-1_mbz-256"
-
-# Verify the upload
-if [ $? -eq 0 ]; then
-    echo "Uploading to Huggingface Hub successful"
-else
-    echo "Failed to upload to Huggingface Hub"
-fi

llama-1B/16_GPUS/dp-1_tp-16_pp-1_mbz-256/config.yaml

@@ -1,90 +0,0 @@
-general:
-  project: bench_cluster
-  seed: 42
-model:
-  ddp_bucket_cap_mb: 25
-  dtype: bfloat16
-  init_method:
-    std: 0.025
-  make_vocab_size_divisible_by: 1
-  model_config:
-    bos_token_id: 1
-    eos_token_id: 2
-    hidden_act: silu
-    hidden_size: 2048
-    initializer_range: 0.02
-    intermediate_size: 4096
-    is_llama_config: true
-    max_position_embeddings: 4096
-    num_attention_heads: 32
-    num_hidden_layers: 24
-    num_key_value_heads: 32
-    pad_token_id: null
-    pretraining_tp: 1
-    rms_norm_eps: 1.0e-05
-    rope_scaling: null
-    rope_theta: 10000.0
-    tie_word_embeddings: true
-    use_cache: true
-    vocab_size: 50257
-optimizer:
-  accumulate_grad_in_fp32: true
-  clip_grad: 1.0
-  learning_rate_scheduler:
-    learning_rate: 0.0001
-    lr_decay_style: linear
-    lr_warmup_style: linear
-    lr_warmup_steps: 1
-    min_decay_lr: 1.0e-05
-  optimizer_factory:
-    adam_beta1: 0.9
-    adam_beta2: 0.95
-    adam_eps: 1.0e-08
-    name: adamW
-    torch_adam_is_fused: true
-  weight_decay: 0.01
-  zero_stage: 1
-parallelism:
-  dp: 1
-  expert_parallel_size: 1
-  pp: 1
-  pp_engine: 1f1b
-  tp: 16
-  tp_linear_async_communication: false
-  tp_mode: REDUCE_SCATTER
-profiler:
-  profiler_export_path: /fsx/ferdinandmom/ferdinand-hf/bench_cluster/results/llama-1B/16_GPUS/dp-1_tp-16_pp-1_mbz-256
-tokenizer:
-  tokenizer_max_length: null
-  tokenizer_name_or_path: openai-community/gpt2
-  tokenizer_revision: null
-data_stages:
-- name: Training Stage
-  start_training_step: 1
-  data:
-    dataset:
-      dataset_overwrite_cache: false
-      dataset_processing_num_proc_per_process: 64
-      hf_dataset_config_name: null
-      hf_dataset_or_datasets: roneneldan/TinyStories
-      hf_dataset_splits: train
-      text_column_name: text
-    num_loading_workers: 32
-    seed: 42
-lighteval: null
-tokens:
-  train_steps: 20
-  val_check_interval: -1
-  batch_accumulation_per_replica: 4
-  limit_test_batches: 0
-  limit_val_batches: 0
-  micro_batch_size: 256
-  sequence_length: 4096
-logging:
-  iteration_step_info_interval: 1
-  log_level: info
-  log_level_replica: info
-checkpoints:
-  checkpoint_interval: 100000
-  checkpoints_path: /dev/null
-  resume_checkpoint_path: null

llama-1B/16_GPUS/dp-1_tp-16_pp-1_mbz-256/status.txt

@@ -1 +0,0 @@
-oom

llama-1B/16_GPUS/dp-1_tp-16_pp-1_mbz-32/bench.slurm

@@ -1,111 +0,0 @@
-#!/bin/bash
-
-#SBATCH --job-name=bench_cluster
-#SBATCH --time=00:59:00
-#SBATCH --partition=hopper-prod
-#SBATCH --nodes=2
-#SBATCH --gres=gpu:8
-#SBATCH --qos=high
-#SBATCH --ntasks-per-node=1
-#SBATCH --cpus-per-task=96
-#SBATCH --exclusive
-#SBATCH --output=/fsx/ferdinandmom/ferdinand-hf/bench_cluster/results/llama-1B/16_GPUS/dp-1_tp-16_pp-1_mbz-32/log.out
-#SBATCH --error=/fsx/ferdinandmom/ferdinand-hf/bench_cluster/results/llama-1B/16_GPUS/dp-1_tp-16_pp-1_mbz-32/log.out
-
-# Function to update status based on squeue output
-update_status() {
-    job_id=$1
-    status_file=$2
-    # For unknown reasons, it doenst update status for pending. It only works for running 
-    while true; do
-        job_status=$(squeue --job $job_id --noheader --format=%T)
-        echo "Job status: $job_status"
-        if [ -z "$job_status" ]; then
-            # Job has finished or is not found
-            break
-        elif [ "$job_status" = "RUNNING" ]; then
-            printf "running" > $status_file
-            break
-        fi
-        sleep 10
-    done
-}
-
-# Misc initializations.
-echo "========================"
-echo "START TIME: $(date)"
-source /fsx/ferdinandmom/miniforge3/etc/profile.d/conda.sh
-conda activate /fsx/ferdinandmom/miniforge3/envs/env-bench-cluster
-echo python3 version = $(python3 --version)
-echo "========================"
-
-# Slurm stuff
-export HOSTNAMES=$(scontrol show hostnames "$SLURM_JOB_NODELIST")
-export MASTER_ADDR=$(scontrol show hostnames "$SLURM_JOB_NODELIST" | head -n 1)
-export MASTER_PORT=$((1024 + RANDOM % 64511))
-
-export TMPDIR=/scratch
-export HF_DATASETS_CACHE="/admin/home/ferdinand_mom/.cache"
-export CUBLAS_WORKSPACE_CONFIG=":4096:8"
-export CUDA_DEVICE_MAX_CONNECTIONS="1"
-
-huggingface-cli login --token $HUGGINGFACE_TOKEN
-
-
-NANOTRON_REPO="/fsx/ferdinandmom/ferdinand-hf/bench_cluster/nanotron"
-CMD="$NANOTRON_REPO/run_train.py --config-file /fsx/ferdinandmom/ferdinand-hf/bench_cluster/results/llama-1B/16_GPUS/dp-1_tp-16_pp-1_mbz-32/config.yaml"
-
-LAUNCHER="torchrun \
-   --nproc_per_node 8 \
-   --nnodes 2 \
-   --rdzv_endpoint ${MASTER_ADDR}:${MASTER_PORT} \
-   --rdzv_backend c10d \
-   --max_restarts 0 \
-   --tee 3 \
-   --node_rank ${SLURM_PROCID}"
-
-# Checkout the bench_cluster branch
-cd $NANOTRON_REPO
-git checkout bench_cluster
-cd ..
-# Get the current job ID
-job_id=${SLURM_JOB_ID}
-
-# Update status to "pending" or "running" in the background
-update_status $job_id /fsx/ferdinandmom/ferdinand-hf/bench_cluster/results/llama-1B/16_GPUS/dp-1_tp-16_pp-1_mbz-32/status.txt &
-
-# Run the main command
-srun -u $LAUNCHER $CMD
-exit_status=$?
-
-# Update status based on the exit status of `srun`
-if [ $exit_status -eq 0 ]; then
-    printf "completed" > /fsx/ferdinandmom/ferdinand-hf/bench_cluster/results/llama-1B/16_GPUS/dp-1_tp-16_pp-1_mbz-32/status.txt
-else
-    if grep -q "OutOfMemoryError" /fsx/ferdinandmom/ferdinand-hf/bench_cluster/results/llama-1B/16_GPUS/dp-1_tp-16_pp-1_mbz-32/log.out; then
-        printf "oom" > /fsx/ferdinandmom/ferdinand-hf/bench_cluster/results/llama-1B/16_GPUS/dp-1_tp-16_pp-1_mbz-32/status.txt
-    elif grep -q " CUDA error: an illegal memory access" /fsx/ferdinandmom/ferdinand-hf/bench_cluster/results/llama-1B/16_GPUS/dp-1_tp-16_pp-1_mbz-32/log.out; then
-        printf "oom" > /fsx/ferdinandmom/ferdinand-hf/bench_cluster/results/llama-1B/16_GPUS/dp-1_tp-16_pp-1_mbz-32/status.txt
-    elif grep -q "Timeout at NCCL" /fsx/ferdinandmom/ferdinand-hf/bench_cluster/results/llama-1B/16_GPUS/dp-1_tp-16_pp-1_mbz-32/log.out; then
-        printf "timeout" > /fsx/ferdinandmom/ferdinand-hf/bench_cluster/results/llama-1B/16_GPUS/dp-1_tp-16_pp-1_mbz-32/status.txt
-    else
-        printf "fail" > /fsx/ferdinandmom/ferdinand-hf/bench_cluster/results/llama-1B/16_GPUS/dp-1_tp-16_pp-1_mbz-32/status.txt
-    fi 
-fi
-
-# Run the report script if the job completed successfully
-if [ $exit_status -eq 0 ]; then
-    python /fsx/ferdinandmom/ferdinand-hf/bench_cluster/main.py report --inp_dir /fsx/ferdinandmom/ferdinand-hf/bench_cluster/results/llama-1B/16_GPUS/dp-1_tp-16_pp-1_mbz-32 --is_logs
-    python /fsx/ferdinandmom/ferdinand-hf/bench_cluster/main.py report --inp_dir /fsx/ferdinandmom/ferdinand-hf/bench_cluster/results/llama-1B/16_GPUS/dp-1_tp-16_pp-1_mbz-32 --is_profiler
-fi
-
-
-# Push to hub the folder using huggingface_cli
-huggingface-cli upload nanotron/bench_cluster /fsx/ferdinandmom/ferdinand-hf/bench_cluster/results/llama-1B/16_GPUS/dp-1_tp-16_pp-1_mbz-32 llama-1B/16_GPUS/dp-1_tp-16_pp-1_mbz-32 --commit-message "Upload llama-1B/16_GPUS/dp-1_tp-16_pp-1_mbz-32"
-
-# Verify the upload
-if [ $? -eq 0 ]; then
-    echo "Uploading to Huggingface Hub successful"
-else
-    echo "Failed to upload to Huggingface Hub"
-fi

llama-1B/16_GPUS/dp-1_tp-16_pp-1_mbz-32/config.yaml

@@ -1,90 +0,0 @@
-general:
-  project: bench_cluster
-  seed: 42
-model:
-  ddp_bucket_cap_mb: 25
-  dtype: bfloat16
-  init_method:
-    std: 0.025
-  make_vocab_size_divisible_by: 1
-  model_config:
-    bos_token_id: 1
-    eos_token_id: 2
-    hidden_act: silu
-    hidden_size: 2048
-    initializer_range: 0.02
-    intermediate_size: 4096
-    is_llama_config: true
-    max_position_embeddings: 4096
-    num_attention_heads: 32
-    num_hidden_layers: 24
-    num_key_value_heads: 32
-    pad_token_id: null
-    pretraining_tp: 1
-    rms_norm_eps: 1.0e-05
-    rope_scaling: null
-    rope_theta: 10000.0
-    tie_word_embeddings: true
-    use_cache: true
-    vocab_size: 50257
-optimizer:
-  accumulate_grad_in_fp32: true
-  clip_grad: 1.0
-  learning_rate_scheduler:
-    learning_rate: 0.0001
-    lr_decay_style: linear
-    lr_warmup_style: linear
-    lr_warmup_steps: 1
-    min_decay_lr: 1.0e-05
-  optimizer_factory:
-    adam_beta1: 0.9
-    adam_beta2: 0.95
-    adam_eps: 1.0e-08
-    name: adamW
-    torch_adam_is_fused: true
-  weight_decay: 0.01
-  zero_stage: 1
-parallelism:
-  dp: 1
-  expert_parallel_size: 1
-  pp: 1
-  pp_engine: 1f1b
-  tp: 16
-  tp_linear_async_communication: false
-  tp_mode: REDUCE_SCATTER
-profiler:
-  profiler_export_path: /fsx/ferdinandmom/ferdinand-hf/bench_cluster/results/llama-1B/16_GPUS/dp-1_tp-16_pp-1_mbz-32
-tokenizer:
-  tokenizer_max_length: null
-  tokenizer_name_or_path: openai-community/gpt2
-  tokenizer_revision: null
-data_stages:
-- name: Training Stage
-  start_training_step: 1
-  data:
-    dataset:
-      dataset_overwrite_cache: false
-      dataset_processing_num_proc_per_process: 64
-      hf_dataset_config_name: null
-      hf_dataset_or_datasets: roneneldan/TinyStories
-      hf_dataset_splits: train
-      text_column_name: text
-    num_loading_workers: 32
-    seed: 42
-lighteval: null
-tokens:
-  train_steps: 20
-  val_check_interval: -1
-  batch_accumulation_per_replica: 32
-  limit_test_batches: 0
-  limit_val_batches: 0
-  micro_batch_size: 32
-  sequence_length: 4096
-logging:
-  iteration_step_info_interval: 1
-  log_level: info
-  log_level_replica: info
-checkpoints:
-  checkpoint_interval: 100000
-  checkpoints_path: /dev/null
-  resume_checkpoint_path: null

llama-1B/16_GPUS/dp-1_tp-16_pp-1_mbz-32/log.out

@@ -1,550 +0,0 @@
-========================
-START TIME: Tue Jul  2 16:31:55 UTC 2024
-python3 version = Python 3.10.14
-========================
-The token has not been saved to the git credentials helper. Pass `add_to_git_credential=True` in this function directly or `--add-to-git-credential` if using via `huggingface-cli` if you want to set the git credential as well.
-Token is valid (permission: write).
-Your token has been saved to /admin/home/ferdinand_mom/.cache/huggingface/token
-Login successful
-Already on 'bench_cluster'
-M	examples/config_tiny_llama.py
-M	examples/config_tiny_llama.yaml
-M	examples/train_tiny_llama.sh
-M	src/nanotron/models/llama.py
-M	src/nanotron/trainer.py
-Your branch is up to date with 'origin/bench_cluster'.
-Job status: RUNNING
-W0702 16:31:57.484000 139796830594880 torch/distributed/run.py:757] 
-W0702 16:31:57.484000 139796830594880 torch/distributed/run.py:757] *****************************************
-W0702 16:31:57.484000 139796830594880 torch/distributed/run.py:757] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. 
-W0702 16:31:57.484000 139796830594880 torch/distributed/run.py:757] *****************************************
-W0702 16:31:57.490000 140386122716992 torch/distributed/run.py:757] 
-W0702 16:31:57.490000 140386122716992 torch/distributed/run.py:757] *****************************************
-W0702 16:31:57.490000 140386122716992 torch/distributed/run.py:757] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. 
-W0702 16:31:57.490000 140386122716992 torch/distributed/run.py:757] *****************************************
-[default0]:07/02/2024 16:32:15 [WARNING|DP=0|PP=0|TP=0|ip-26-0-169-239]: [Vocab Size Padding] Padded vocab (size: 50257) with 15 dummy tokens (new size: 50272)
-[default0]:07/02/2024 16:32:15 [INFO|DP=0|PP=0|TP=0|ip-26-0-169-239]: Config:
-[default0]:07/02/2024 16:32:15 [INFO|DP=0|PP=0|TP=0|ip-26-0-169-239]: Config(general=GeneralArgs(project='bench_cluster',
-[default0]:07/02/2024 16:32:15 [INFO|DP=0|PP=0|TP=0|ip-26-0-169-239]:                            run='%date_%jobid',
-[default0]:07/02/2024 16:32:15 [INFO|DP=0|PP=0|TP=0|ip-26-0-169-239]:                            seed=42,
-[default0]:07/02/2024 16:32:15 [INFO|DP=0|PP=0|TP=0|ip-26-0-169-239]:                            step=None,
-[default0]:07/02/2024 16:32:15 [INFO|DP=0|PP=0|TP=0|ip-26-0-169-239]:                            consumed_train_samples=None,
-[default0]:07/02/2024 16:32:15 [INFO|DP=0|PP=0|TP=0|ip-26-0-169-239]:                            benchmark_csv_path=None,
-[default0]:07/02/2024 16:32:15 [INFO|DP=0|PP=0|TP=0|ip-26-0-169-239]:                            ignore_sanity_checks=True),
-[default0]:07/02/2024 16:32:15 [INFO|DP=0|PP=0|TP=0|ip-26-0-169-239]:        parallelism=ParallelismArgs(dp=1,
-[default0]:07/02/2024 16:32:15 [INFO|DP=0|PP=0|TP=0|ip-26-0-169-239]:                                    pp=1,
-[default0]:07/02/2024 16:32:15 [INFO|DP=0|PP=0|TP=0|ip-26-0-169-239]:                                    tp=16,
-[default0]:07/02/2024 16:32:15 [INFO|DP=0|PP=0|TP=0|ip-26-0-169-239]:                                    pp_engine=<nanotron.parallel.pipeline_parallel.engine.OneForwardOneBackwardPipelineEngine object at 0x7f55d6098790>,
-[default0]:07/02/2024 16:32:15 [INFO|DP=0|PP=0|TP=0|ip-26-0-169-239]:                                    tp_mode=<TensorParallelLinearMode.REDUCE_SCATTER: 2>,
-[default0]:07/02/2024 16:32:15 [INFO|DP=0|PP=0|TP=0|ip-26-0-169-239]:                                    tp_linear_async_communication=False,
-[default0]:07/02/2024 16:32:15 [INFO|DP=0|PP=0|TP=0|ip-26-0-169-239]:                                    expert_parallel_size=1),
-[default0]:07/02/2024 16:32:15 [INFO|DP=0|PP=0|TP=0|ip-26-0-169-239]:        model=ModelArgs(model_config=LlamaConfig(bos_token_id=1,
-[default0]:07/02/2024 16:32:15 [INFO|DP=0|PP=0|TP=0|ip-26-0-169-239]:                                                 eos_token_id=2,
-[default0]:07/02/2024 16:32:15 [INFO|DP=0|PP=0|TP=0|ip-26-0-169-239]:                                                 hidden_act='silu',
-[default0]:07/02/2024 16:32:15 [INFO|DP=0|PP=0|TP=0|ip-26-0-169-239]:                                                 hidden_size=2048,
-[default0]:07/02/2024 16:32:15 [INFO|DP=0|PP=0|TP=0|ip-26-0-169-239]:                                                 initializer_range=0.02,
-[default0]:07/02/2024 16:32:15 [INFO|DP=0|PP=0|TP=0|ip-26-0-169-239]:                                                 intermediate_size=4096,
-[default0]:07/02/2024 16:32:15 [INFO|DP=0|PP=0|TP=0|ip-26-0-169-239]:                                                 is_llama_config=True,
-[default0]:07/02/2024 16:32:15 [INFO|DP=0|PP=0|TP=0|ip-26-0-169-239]:                                                 max_position_embeddings=4096,
-[default0]:07/02/2024 16:32:15 [INFO|DP=0|PP=0|TP=0|ip-26-0-169-239]:                                                 num_attention_heads=32,
-[default0]:07/02/2024 16:32:15 [INFO|DP=0|PP=0|TP=0|ip-26-0-169-239]:                                                 num_hidden_layers=24,
-[default0]:07/02/2024 16:32:15 [INFO|DP=0|PP=0|TP=0|ip-26-0-169-239]:                                                 num_key_value_heads=32,
-[default0]:07/02/2024 16:32:15 [INFO|DP=0|PP=0|TP=0|ip-26-0-169-239]:                                                 pad_token_id=None,
-[default0]:07/02/2024 16:32:15 [INFO|DP=0|PP=0|TP=0|ip-26-0-169-239]:                                                 pretraining_tp=1,
-[default0]:07/02/2024 16:32:15 [INFO|DP=0|PP=0|TP=0|ip-26-0-169-239]:                                                 rms_norm_eps=1e-05,
-[default0]:07/02/2024 16:32:15 [INFO|DP=0|PP=0|TP=0|ip-26-0-169-239]:                                                 rope_scaling=None,
-[default0]:07/02/2024 16:32:15 [INFO|DP=0|PP=0|TP=0|ip-26-0-169-239]:                                                 rope_theta=10000.0,
-[default0]:07/02/2024 16:32:15 [INFO|DP=0|PP=0|TP=0|ip-26-0-169-239]:                                                 tie_word_embeddings=True,
-[default0]:07/02/2024 16:32:15 [INFO|DP=0|PP=0|TP=0|ip-26-0-169-239]:                                                 use_cache=True,
-[default0]:07/02/2024 16:32:15 [INFO|DP=0|PP=0|TP=0|ip-26-0-169-239]:                                                 vocab_size=50272),
-[default0]:07/02/2024 16:32:15 [INFO|DP=0|PP=0|TP=0|ip-26-0-169-239]:                        init_method=RandomInit(std=0.025),
-[default0]:07/02/2024 16:32:15 [INFO|DP=0|PP=0|TP=0|ip-26-0-169-239]:                        dtype=torch.bfloat16,
-[default0]:07/02/2024 16:32:15 [INFO|DP=0|PP=0|TP=0|ip-26-0-169-239]:                        make_vocab_size_divisible_by=1,
-[default0]:07/02/2024 16:32:15 [INFO|DP=0|PP=0|TP=0|ip-26-0-169-239]:                        ddp_bucket_cap_mb=25),
-[default0]:07/02/2024 16:32:15 [INFO|DP=0|PP=0|TP=0|ip-26-0-169-239]:        tokenizer=TokenizerArgs(tokenizer_name_or_path='openai-community/gpt2',
-[default0]:07/02/2024 16:32:15 [INFO|DP=0|PP=0|TP=0|ip-26-0-169-239]:                                tokenizer_revision=None,
-[default0]:07/02/2024 16:32:15 [INFO|DP=0|PP=0|TP=0|ip-26-0-169-239]:                                tokenizer_max_length=None),
-[default0]:07/02/2024 16:32:15 [INFO|DP=0|PP=0|TP=0|ip-26-0-169-239]:        checkpoints=CheckpointsArgs(checkpoints_path=Path('/dev/null'),
-[default0]:07/02/2024 16:32:15 [INFO|DP=0|PP=0|TP=0|ip-26-0-169-239]:                                    checkpoint_interval=100000,
-[default0]:07/02/2024 16:32:15 [INFO|DP=0|PP=0|TP=0|ip-26-0-169-239]:                                    save_initial_state=False,
-[default0]:07/02/2024 16:32:15 [INFO|DP=0|PP=0|TP=0|ip-26-0-169-239]:                                    resume_checkpoint_path=None,
-[default0]:07/02/2024 16:32:15 [INFO|DP=0|PP=0|TP=0|ip-26-0-169-239]:                                    checkpoints_path_is_shared_file_system=False),
-[default0]:07/02/2024 16:32:15 [INFO|DP=0|PP=0|TP=0|ip-26-0-169-239]:        logging=LoggingArgs(log_level='info',
-[default0]:07/02/2024 16:32:15 [INFO|DP=0|PP=0|TP=0|ip-26-0-169-239]:                            log_level_replica='info',
-[default0]:07/02/2024 16:32:15 [INFO|DP=0|PP=0|TP=0|ip-26-0-169-239]:                            iteration_step_info_interval=1),
-[default0]:07/02/2024 16:32:15 [INFO|DP=0|PP=0|TP=0|ip-26-0-169-239]:        tokens=TokensArgs(sequence_length=4096,
-[default0]:07/02/2024 16:32:15 [INFO|DP=0|PP=0|TP=0|ip-26-0-169-239]:                          train_steps=20,
-[default0]:07/02/2024 16:32:15 [INFO|DP=0|PP=0|TP=0|ip-26-0-169-239]:                          micro_batch_size=32,
-[default0]:07/02/2024 16:32:15 [INFO|DP=0|PP=0|TP=0|ip-26-0-169-239]:                          batch_accumulation_per_replica=32,
-[default0]:07/02/2024 16:32:15 [INFO|DP=0|PP=0|TP=0|ip-26-0-169-239]:                          val_check_interval=-1,
-[default0]:07/02/2024 16:32:15 [INFO|DP=0|PP=0|TP=0|ip-26-0-169-239]:                          limit_val_batches=0,
-[default0]:07/02/2024 16:32:15 [INFO|DP=0|PP=0|TP=0|ip-26-0-169-239]:                          limit_test_batches=0),
-[default0]:07/02/2024 16:32:15 [INFO|DP=0|PP=0|TP=0|ip-26-0-169-239]:        optimizer=OptimizerArgs(optimizer_factory=AdamWOptimizerArgs(adam_eps=1e-08,
-[default0]:07/02/2024 16:32:15 [INFO|DP=0|PP=0|TP=0|ip-26-0-169-239]:                                                                     adam_beta1=0.9,
-[default0]:07/02/2024 16:32:15 [INFO|DP=0|PP=0|TP=0|ip-26-0-169-239]:                                                                     adam_beta2=0.95,
-[default0]:07/02/2024 16:32:15 [INFO|DP=0|PP=0|TP=0|ip-26-0-169-239]:                                                                     torch_adam_is_fused=True,
-[default0]:07/02/2024 16:32:15 [INFO|DP=0|PP=0|TP=0|ip-26-0-169-239]:                                                                     name='adamW'),
-[default0]:07/02/2024 16:32:15 [INFO|DP=0|PP=0|TP=0|ip-26-0-169-239]:                                zero_stage=1,
-[default0]:07/02/2024 16:32:15 [INFO|DP=0|PP=0|TP=0|ip-26-0-169-239]:                                weight_decay=0.01,
-[default0]:07/02/2024 16:32:15 [INFO|DP=0|PP=0|TP=0|ip-26-0-169-239]:                                clip_grad=1.0,
-[default0]:07/02/2024 16:32:15 [INFO|DP=0|PP=0|TP=0|ip-26-0-169-239]:                                accumulate_grad_in_fp32=True,
-[default0]:07/02/2024 16:32:15 [INFO|DP=0|PP=0|TP=0|ip-26-0-169-239]:                                learning_rate_scheduler=LRSchedulerArgs(learning_rate=0.0001,
-[default0]:07/02/2024 16:32:15 [INFO|DP=0|PP=0|TP=0|ip-26-0-169-239]:                                                                        lr_warmup_steps=1,
-[default0]:07/02/2024 16:32:15 [INFO|DP=0|PP=0|TP=0|ip-26-0-169-239]:                                                                        lr_warmup_style='linear',
-[default0]:07/02/2024 16:32:15 [INFO|DP=0|PP=0|TP=0|ip-26-0-169-239]:                                                                        lr_decay_style='linear',
-[default0]:07/02/2024 16:32:15 [INFO|DP=0|PP=0|TP=0|ip-26-0-169-239]:                                                                        lr_decay_steps=19,
-[default0]:07/02/2024 16:32:15 [INFO|DP=0|PP=0|TP=0|ip-26-0-169-239]:                                                                        lr_decay_starting_step=None,
-[default0]:07/02/2024 16:32:15 [INFO|DP=0|PP=0|TP=0|ip-26-0-169-239]:                                                                        min_decay_lr=1e-05)),
-[default0]:07/02/2024 16:32:15 [INFO|DP=0|PP=0|TP=0|ip-26-0-169-239]:        data_stages=[DatasetStageArgs(name='Training Stage',
-[default0]:07/02/2024 16:32:15 [INFO|DP=0|PP=0|TP=0|ip-26-0-169-239]:                                      start_training_step=1,
-[default0]:07/02/2024 16:32:15 [INFO|DP=0|PP=0|TP=0|ip-26-0-169-239]:                                      data=DataArgs(dataset=PretrainDatasetsArgs(hf_dataset_or_datasets='roneneldan/TinyStories',
-[default0]:07/02/2024 16:32:15 [INFO|DP=0|PP=0|TP=0|ip-26-0-169-239]:                                                                                 hf_dataset_splits='train',
-[default0]:07/02/2024 16:32:15 [INFO|DP=0|PP=0|TP=0|ip-26-0-169-239]:                                                                                 hf_dataset_config_name=None,
-[default0]:07/02/2024 16:32:15 [INFO|DP=0|PP=0|TP=0|ip-26-0-169-239]:                                                                                 dataset_processing_num_proc_per_process=64,
-[default0]:07/02/2024 16:32:15 [INFO|DP=0|PP=0|TP=0|ip-26-0-169-239]:                                                                                 dataset_overwrite_cache=False,
-[default0]:07/02/2024 16:32:15 [INFO|DP=0|PP=0|TP=0|ip-26-0-169-239]:                                                                                 text_column_name='text'),
-[default0]:07/02/2024 16:32:15 [INFO|DP=0|PP=0|TP=0|ip-26-0-169-239]:                                                    seed=42,
-[default0]:07/02/2024 16:32:15 [INFO|DP=0|PP=0|TP=0|ip-26-0-169-239]:                                                    num_loading_workers=32))],
-[default0]:07/02/2024 16:32:15 [INFO|DP=0|PP=0|TP=0|ip-26-0-169-239]:        profiler=ProfilerArgs(profiler_export_path=Path('/fsx/ferdinandmom/ferdinand-hf/bench_cluster/results/llama-1B/16_GPUS/dp-1_tp-16_pp-1_mbz-32')),
-[default0]:07/02/2024 16:32:15 [INFO|DP=0|PP=0|TP=0|ip-26-0-169-239]:        lighteval=None)
-[default0]:07/02/2024 16:32:15 [INFO|DP=0|PP=0|TP=0|ip-26-0-169-239]: Model Config:
-[default0]:07/02/2024 16:32:15 [INFO|DP=0|PP=0|TP=0|ip-26-0-169-239]: LlamaConfig(bos_token_id=1,
-[default0]:07/02/2024 16:32:15 [INFO|DP=0|PP=0|TP=0|ip-26-0-169-239]:             eos_token_id=2,
-[default0]:07/02/2024 16:32:15 [INFO|DP=0|PP=0|TP=0|ip-26-0-169-239]:             hidden_act='silu',
-[default0]:07/02/2024 16:32:15 [INFO|DP=0|PP=0|TP=0|ip-26-0-169-239]:             hidden_size=2048,
-[default0]:07/02/2024 16:32:15 [INFO|DP=0|PP=0|TP=0|ip-26-0-169-239]:             initializer_range=0.02,
-[default0]:07/02/2024 16:32:15 [INFO|DP=0|PP=0|TP=0|ip-26-0-169-239]:             intermediate_size=4096,
-[default0]:07/02/2024 16:32:15 [INFO|DP=0|PP=0|TP=0|ip-26-0-169-239]:             is_llama_config=True,
-[default0]:07/02/2024 16:32:15 [INFO|DP=0|PP=0|TP=0|ip-26-0-169-239]:             max_position_embeddings=4096,
-[default0]:07/02/2024 16:32:15 [INFO|DP=0|PP=0|TP=0|ip-26-0-169-239]:             num_attention_heads=32,
-[default0]:07/02/2024 16:32:15 [INFO|DP=0|PP=0|TP=0|ip-26-0-169-239]:             num_hidden_layers=24,
-[default0]:07/02/2024 16:32:15 [INFO|DP=0|PP=0|TP=0|ip-26-0-169-239]:             num_key_value_heads=32,
-[default0]:07/02/2024 16:32:15 [INFO|DP=0|PP=0|TP=0|ip-26-0-169-239]:             pad_token_id=None,
-[default0]:07/02/2024 16:32:15 [INFO|DP=0|PP=0|TP=0|ip-26-0-169-239]:             pretraining_tp=1,
-[default0]:07/02/2024 16:32:15 [INFO|DP=0|PP=0|TP=0|ip-26-0-169-239]:             rms_norm_eps=1e-05,
-[default0]:07/02/2024 16:32:15 [INFO|DP=0|PP=0|TP=0|ip-26-0-169-239]:             rope_scaling=None,
-[default0]:07/02/2024 16:32:15 [INFO|DP=0|PP=0|TP=0|ip-26-0-169-239]:             rope_theta=10000.0,
-[default0]:07/02/2024 16:32:15 [INFO|DP=0|PP=0|TP=0|ip-26-0-169-239]:             tie_word_embeddings=True,
-[default0]:07/02/2024 16:32:15 [INFO|DP=0|PP=0|TP=0|ip-26-0-169-239]:             use_cache=True,
-[default0]:07/02/2024 16:32:15 [INFO|DP=0|PP=0|TP=0|ip-26-0-169-239]:             vocab_size=50272)
-[default0]:07/02/2024 16:32:15 [INFO|DP=0|PP=0|TP=0|ip-26-0-169-239]: Building model..
-[default0]:07/02/2024 16:32:15 [INFO|DP=0|PP=0|TP=0|ip-26-0-169-239]: Setting PP block ranks...
-[default3]:07/02/2024 16:32:32 [INFO|DP=0|PP=0|TP=3|ip-26-0-169-239]: Local number of parameters: 69.4M (132.46MiB)
-[default3]:07/02/2024 16:32:32 [INFO|DP=0|PP=0|TP=3|ip-26-0-169-239]: [After model building] Memory usage: 159.71MiB. Peak allocated: 174.02MiB Peak reserved: 178.00MiB
-[default4]:07/02/2024 16:32:32 [INFO|DP=0|PP=0|TP=4|ip-26-0-169-239]: Local number of parameters: 69.4M (132.46MiB)
-[default4]:07/02/2024 16:32:32 [INFO|DP=0|PP=0|TP=4|ip-26-0-169-239]: [After model building] Memory usage: 159.71MiB. Peak allocated: 174.02MiB Peak reserved: 178.00MiB
-[default1]:07/02/2024 16:32:32 [INFO|DP=0|PP=0|TP=1|ip-26-0-169-239]: Local number of parameters: 69.4M (132.46MiB)
-[default4]:07/02/2024 16:32:32 [INFO|DP=0|PP=0|TP=4|ip-26-0-169-239]: No checkpoint path provided.
-[default3]:07/02/2024 16:32:32 [INFO|DP=0|PP=0|TP=3|ip-26-0-169-239]: No checkpoint path provided.
-[default1]:07/02/2024 16:32:32 [INFO|DP=0|PP=0|TP=1|ip-26-0-169-239]: [After model building] Memory usage: 159.71MiB. Peak allocated: 174.02MiB Peak reserved: 178.00MiB
-[default1]:07/02/2024 16:32:32 [INFO|DP=0|PP=0|TP=1|ip-26-0-169-239]: No checkpoint path provided.
-[default5]:07/02/2024 16:32:32 [INFO|DP=0|PP=0|TP=5|ip-26-0-169-239]: Local number of parameters: 69.4M (132.46MiB)
-[default5]:07/02/2024 16:32:32 [INFO|DP=0|PP=0|TP=5|ip-26-0-169-239]: [After model building] Memory usage: 159.71MiB. Peak allocated: 174.02MiB Peak reserved: 178.00MiB
-[default5]:07/02/2024 16:32:32 [INFO|DP=0|PP=0|TP=5|ip-26-0-169-239]: No checkpoint path provided.
-[default7]:07/02/2024 16:32:32 [INFO|DP=0|PP=0|TP=7|ip-26-0-169-239]: Local number of parameters: 69.4M (132.46MiB)
-[default7]:07/02/2024 16:32:32 [INFO|DP=0|PP=0|TP=7|ip-26-0-169-239]: [After model building] Memory usage: 159.71MiB. Peak allocated: 174.02MiB Peak reserved: 178.00MiB
-[default7]:07/02/2024 16:32:32 [INFO|DP=0|PP=0|TP=7|ip-26-0-169-239]: No checkpoint path provided.
-[default2]:07/02/2024 16:32:32 [INFO|DP=0|PP=0|TP=2|ip-26-0-169-239]: Local number of parameters: 69.4M (132.46MiB)
-[default2]:07/02/2024 16:32:32 [INFO|DP=0|PP=0|TP=2|ip-26-0-169-239]: [After model building] Memory usage: 159.71MiB. Peak allocated: 174.02MiB Peak reserved: 178.00MiB
-[default2]:07/02/2024 16:32:32 [INFO|DP=0|PP=0|TP=2|ip-26-0-169-239]: No checkpoint path provided.
-[default0]:07/02/2024 16:32:32 [INFO|DP=0|PP=0|TP=0|ip-26-0-169-239]: Total number of parameters: 1.11G (2119.44MiB)
-[default0]:07/02/2024 16:32:32 [INFO|DP=0|PP=0|TP=0|ip-26-0-169-239]: Local number of parameters: 69.4M (132.46MiB)
-[default0]:07/02/2024 16:32:32 [INFO|DP=0|PP=0|TP=0|ip-26-0-169-239]: [After model building] Memory usage: 159.71MiB. Peak allocated: 174.02MiB Peak reserved: 178.00MiB
-[default0]:07/02/2024 16:32:32 [INFO|DP=0|PP=0|TP=0|ip-26-0-169-239]: No checkpoint path provided.
-[default0]:07/02/2024 16:32:32 [INFO|DP=0|PP=0|TP=0|ip-26-0-169-239]: Parametrizing model parameters using StandardParametrizator
-[default0]:07/02/2024 16:32:32 [INFO|DP=0|PP=0|TP=0|ip-26-0-169-239]: [Optimizer Building] Using LearningRateForSP as learning rate
-[default0]:07/02/2024 16:32:32 [INFO|DP=0|PP=0|TP=0|ip-26-0-169-239]: [ZeRO sharding] Size of optimizer params per rank:
-[default0]:07/02/2024 16:32:32 [INFO|DP=0|PP=0|TP=0|ip-26-0-169-239]: [ZeRO sharding] DP Rank 0 has 69.4M out of 69.4M (100.00%) params' optimizer states
-[default6]:07/02/2024 16:32:32 [INFO|DP=0|PP=0|TP=6|ip-26-0-169-239]: Local number of parameters: 69.4M (132.46MiB)
-[default6]:07/02/2024 16:32:32 [INFO|DP=0|PP=0|TP=6|ip-26-0-169-239]: [After model building] Memory usage: 159.71MiB. Peak allocated: 174.02MiB Peak reserved: 178.00MiB
-[default6]:07/02/2024 16:32:32 [INFO|DP=0|PP=0|TP=6|ip-26-0-169-239]: No checkpoint path provided.
-[default3]:07/02/2024 16:32:32 [INFO|DP=0|PP=0|TP=11|ip-26-0-169-247]: Local number of parameters: 69.4M (132.46MiB)
-[default3]:07/02/2024 16:32:32 [INFO|DP=0|PP=0|TP=11|ip-26-0-169-247]: [After model building] Memory usage: 159.71MiB. Peak allocated: 174.02MiB Peak reserved: 178.00MiB
-[default3]:07/02/2024 16:32:32 [INFO|DP=0|PP=0|TP=11|ip-26-0-169-247]: No checkpoint path provided.
-[default1]:07/02/2024 16:32:32 [INFO|DP=0|PP=0|TP=9|ip-26-0-169-247]: Local number of parameters: 69.4M (132.46MiB)
-[default1]:07/02/2024 16:32:32 [INFO|DP=0|PP=0|TP=9|ip-26-0-169-247]: [After model building] Memory usage: 159.71MiB. Peak allocated: 174.02MiB Peak reserved: 178.00MiB
-[default1]:07/02/2024 16:32:32 [INFO|DP=0|PP=0|TP=9|ip-26-0-169-247]: No checkpoint path provided.
-[default5]:07/02/2024 16:32:32 [INFO|DP=0|PP=0|TP=13|ip-26-0-169-247]: Local number of parameters: 69.4M (132.46MiB)
-[default5]:07/02/2024 16:32:32 [INFO|DP=0|PP=0|TP=13|ip-26-0-169-247]: [After model building] Memory usage: 159.71MiB. Peak allocated: 174.02MiB Peak reserved: 178.00MiB
-[default2]:07/02/2024 16:32:32 [INFO|DP=0|PP=0|TP=10|ip-26-0-169-247]: Local number of parameters: 69.4M (132.46MiB)
-[default6]:07/02/2024 16:32:32 [INFO|DP=0|PP=0|TP=14|ip-26-0-169-247]: Local number of parameters: 69.4M (132.46MiB)
-[default6]:07/02/2024 16:32:32 [INFO|DP=0|PP=0|TP=14|ip-26-0-169-247]: [After model building] Memory usage: 159.71MiB. Peak allocated: 174.02MiB Peak reserved: 178.00MiB
-[default5]:07/02/2024 16:32:32 [INFO|DP=0|PP=0|TP=13|ip-26-0-169-247]: No checkpoint path provided.
-[default4]:07/02/2024 16:32:32 [INFO|DP=0|PP=0|TP=12|ip-26-0-169-247]: Local number of parameters: 69.4M (132.46MiB)
-[default4]:07/02/2024 16:32:32 [INFO|DP=0|PP=0|TP=12|ip-26-0-169-247]: [After model building] Memory usage: 159.71MiB. Peak allocated: 174.02MiB Peak reserved: 178.00MiB
-[default2]:07/02/2024 16:32:32 [INFO|DP=0|PP=0|TP=10|ip-26-0-169-247]: [After model building] Memory usage: 159.71MiB. Peak allocated: 174.02MiB Peak reserved: 178.00MiB
-[default2]:07/02/2024 16:32:32 [INFO|DP=0|PP=0|TP=10|ip-26-0-169-247]: No checkpoint path provided.
-[default6]:07/02/2024 16:32:32 [INFO|DP=0|PP=0|TP=14|ip-26-0-169-247]: No checkpoint path provided.
-[default4]:07/02/2024 16:32:32 [INFO|DP=0|PP=0|TP=12|ip-26-0-169-247]: No checkpoint path provided.
-[default0]:07/02/2024 16:32:32 [INFO|DP=0|PP=0|TP=8|ip-26-0-169-247]: Local number of parameters: 69.4M (132.46MiB)
-[default0]:07/02/2024 16:32:32 [INFO|DP=0|PP=0|TP=8|ip-26-0-169-247]: [After model building] Memory usage: 159.71MiB. Peak allocated: 174.02MiB Peak reserved: 178.00MiB
-[default0]:07/02/2024 16:32:32 [INFO|DP=0|PP=0|TP=8|ip-26-0-169-247]: No checkpoint path provided.
-[default7]:07/02/2024 16:32:32 [INFO|DP=0|PP=0|TP=15|ip-26-0-169-247]: Local number of parameters: 69.4M (132.46MiB)
-[default7]:07/02/2024 16:32:32 [INFO|DP=0|PP=0|TP=15|ip-26-0-169-247]: [After model building] Memory usage: 159.71MiB. Peak allocated: 174.02MiB Peak reserved: 178.00MiB
-[default7]:07/02/2024 16:32:32 [INFO|DP=0|PP=0|TP=15|ip-26-0-169-247]: No checkpoint path provided.
-[default0]:07/02/2024 16:32:33 [INFO|DP=0|PP=0|TP=0|ip-26-0-169-239]: [Training Plan] Stage Training Stage has 19 remaining training steps and has consumed 0 samples
-[default0]:07/02/2024 16:32:33 [INFO|DP=0|PP=0|TP=0|ip-26-0-169-239]: Using `datasets` library
-[default0]:07/02/2024 16:32:33 [INFO|DP=0|PP=0|TP=0|ip-26-0-169-239]: Loading tokenizer from openai-community/gpt2 and transformers/hf_hub versions ('4.41.2', '0.23.4')
-[default0]:07/02/2024 16:32:33 [WARNING|DP=0|PP=0|TP=0|ip-26-0-169-239]: Repo card metadata block was not found. Setting CardData to empty.
-[default0]:Repo card metadata block was not found. Setting CardData to empty.
-[default0]:07/02/2024 16:32:34 [INFO|DP=0|PP=0|TP=0|ip-26-0-169-239]: [Training Plan] There are 1 training stages 
-[default0]:07/02/2024 16:32:34 [INFO|DP=0|PP=0|TP=0|ip-26-0-169-239]: [Stage Training Stage] start from step 1 
-[default0]:07/02/2024 16:32:34 [INFO|DP=0|PP=0|TP=0|ip-26-0-169-239]: 
-[default0]:07/02/2024 16:32:34 [INFO|DP=0|PP=0|TP=0|ip-26-0-169-239]: [Start training] datetime: 2024-07-02 16:32:34.409740 | mbs: 32 | grad_accum: 32 | global_batch_size: 1024 | sequence_length: 4096 | train_steps: 20 | start_iteration_step: 0 | consumed_train_samples: 0
-[default0]:07/02/2024 16:32:34 [INFO|DP=0|PP=0|TP=0|ip-26-0-169-239]: Resuming training from stage Training Stage, it has trained for 0 samples and has 19 remaining train steps
-[default0]:07/02/2024 16:32:34 [INFO|DP=0|PP=0|TP=0|ip-26-0-169-239]:  Memory usage: 689.57MiB. Peak allocated 689.57MiB. Peak reserved: 710.00MiB
-[default4]:07/02/2024 16:32:34 [WARNING|DP=0|PP=0|TP=4|ip-26-0-169-239]: Repo card metadata block was not found. Setting CardData to empty.
-[default2]:07/02/2024 16:32:34 [WARNING|DP=0|PP=0|TP=2|ip-26-0-169-239]: Repo card metadata block was not found. Setting CardData to empty.
-[default2]:Repo card metadata block was not found. Setting CardData to empty.
-[default0]:Repo card metadata block was not found. Setting CardData to empty.
-[default5]:07/02/2024 16:32:34 [WARNING|DP=0|PP=0|TP=13|ip-26-0-169-247]: Repo card metadata block was not found. Setting CardData to empty.
-[default2]:07/02/2024 16:32:34 [WARNING|DP=0|PP=0|TP=10|ip-26-0-169-247]: Repo card metadata block was not found. Setting CardData to empty.
-[default1]:07/02/2024 16:32:34 [WARNING|DP=0|PP=0|TP=9|ip-26-0-169-247]: Repo card metadata block was not found. Setting CardData to empty.
-[default4]:Repo card metadata block was not found. Setting CardData to empty.
-[default0]:07/02/2024 16:32:34 [WARNING|DP=0|PP=0|TP=8|ip-26-0-169-247]: Repo card metadata block was not found. Setting CardData to empty.
-[default3]:Repo card metadata block was not found. Setting CardData to empty.
-[default1]:Repo card metadata block was not found. Setting CardData to empty.
-[default5]:Repo card metadata block was not found. Setting CardData to empty.
-[default2]:Repo card metadata block was not found. Setting CardData to empty.
-[default3]:07/02/2024 16:32:34 [WARNING|DP=0|PP=0|TP=3|ip-26-0-169-239]: Repo card metadata block was not found. Setting CardData to empty.
-[default1]:07/02/2024 16:32:34 [WARNING|DP=0|PP=0|TP=1|ip-26-0-169-239]: Repo card metadata block was not found. Setting CardData to empty.
-[default5]:07/02/2024 16:32:34 [WARNING|DP=0|PP=0|TP=5|ip-26-0-169-239]: Repo card metadata block was not found. Setting CardData to empty.
-[default7]:07/02/2024 16:32:34 [WARNING|DP=0|PP=0|TP=7|ip-26-0-169-239]: Repo card metadata block was not found. Setting CardData to empty.
-[default3]:Repo card metadata block was not found. Setting CardData to empty.
-[default1]:Repo card metadata block was not found. Setting CardData to empty.
-[default6]:07/02/2024 16:32:34 [WARNING|DP=0|PP=0|TP=6|ip-26-0-169-239]: Repo card metadata block was not found. Setting CardData to empty.
-[default6]:Repo card metadata block was not found. Setting CardData to empty.
-[default3]:07/02/2024 16:32:34 [WARNING|DP=0|PP=0|TP=11|ip-26-0-169-247]: Repo card metadata block was not found. Setting CardData to empty.
-[default6]:07/02/2024 16:32:34 [WARNING|DP=0|PP=0|TP=14|ip-26-0-169-247]: Repo card metadata block was not found. Setting CardData to empty.
-[default4]:07/02/2024 16:32:34 [WARNING|DP=0|PP=0|TP=12|ip-26-0-169-247]: Repo card metadata block was not found. Setting CardData to empty.
-[default5]:Repo card metadata block was not found. Setting CardData to empty.
-[default7]:Repo card metadata block was not found. Setting CardData to empty.
-[default4]:Repo card metadata block was not found. Setting CardData to empty.
-[default7]:07/02/2024 16:32:34 [WARNING|DP=0|PP=0|TP=15|ip-26-0-169-247]: Repo card metadata block was not found. Setting CardData to empty.
-[default6]:Repo card metadata block was not found. Setting CardData to empty.
-[default7]:Repo card metadata block was not found. Setting CardData to empty.
-[default7]:[rank7]: OSError: [Errno 122] Disk quota exceeded
-[default7]:
-[default7]:[rank7]: During handling of the above exception, another exception occurred:
-[default7]:
-[default7]:[rank7]: Traceback (most recent call last):
-[default7]:[rank7]:   File "/fsx/ferdinandmom/ferdinand-hf/bench_cluster/nanotron/run_train.py", line 237, in <module>
-[default7]:[rank7]:     trainer.train(dataloader)
-[default7]:[rank7]:   File "/fsx/ferdinandmom/ferdinand-hf/bench_cluster/nanotron/src/nanotron/trainer.py", line 429, in train
-[default7]:[rank7]:     outputs, loss_avg = self.training_step(dataloader=self.current_dataloader)
-[default7]:[rank7]:   File "/fsx/ferdinandmom/ferdinand-hf/bench_cluster/nanotron/src/nanotron/trainer.py", line 462, in training_step
-[default7]:[rank7]:     outputs = self.pipeline_engine.train_batch_iter(
-[default7]:[rank7]:   File "/fsx/ferdinandmom/ferdinand-hf/bench_cluster/nanotron/src/nanotron/parallel/pipeline_parallel/engine.py", line 278, in train_batch_iter
-[default7]:[rank7]:     output = self.forward(context=context, state=state, micro_batch=micro_batch, model=model)
-[default7]:[rank7]:   File "/fsx/ferdinandmom/ferdinand-hf/bench_cluster/nanotron/src/nanotron/parallel/pipeline_parallel/engine.py", line 44, in forward
-[default7]:[rank7]:     output = model(**micro_batch)
-[default7]:[rank7]:   File "/fsx/ferdinandmom/miniforge3/envs/env-bench-cluster/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1532, in _wrapped_call_impl
-[default7]:[rank7]:     return self._call_impl(*args, **kwargs)
-[default7]:[rank7]:   File "/fsx/ferdinandmom/miniforge3/envs/env-bench-cluster/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1541, in _call_impl
-[default7]:[rank7]:     return forward_call(*args, **kwargs)
-[default7]:[rank7]:   File "/fsx/ferdinandmom/ferdinand-hf/bench_cluster/nanotron/src/nanotron/models/llama.py", line 891, in forward
-[default7]:[rank7]:     sharded_logits = self.model(
-[default7]:[rank7]:   File "/fsx/ferdinandmom/miniforge3/envs/env-bench-cluster/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1532, in _wrapped_call_impl
-[default7]:[rank7]:     return self._call_impl(*args, **kwargs)
-[default7]:[rank7]:   File "/fsx/ferdinandmom/miniforge3/envs/env-bench-cluster/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1541, in _call_impl
-[default7]:[rank7]:     return forward_call(*args, **kwargs)
-[default7]:[rank7]:   File "/fsx/ferdinandmom/ferdinand-hf/bench_cluster/nanotron/src/nanotron/models/llama.py", line 764, in forward
-[default7]:[rank7]:     return self.forward_with_hidden_states(input_ids=input_ids, input_mask=input_mask)[0]
-[default7]:[rank7]:   File "/fsx/ferdinandmom/ferdinand-hf/bench_cluster/nanotron/src/nanotron/models/llama.py", line 780, in forward_with_hidden_states
-[default7]:[rank7]:     hidden_encoder_states = encoder_block(**hidden_encoder_states)
-[default7]:[rank7]:   File "/fsx/ferdinandmom/miniforge3/envs/env-bench-cluster/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1532, in _wrapped_call_impl
-[default7]:[rank7]:     return self._call_impl(*args, **kwargs)
-[default7]:[rank7]:   File "/fsx/ferdinandmom/miniforge3/envs/env-bench-cluster/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1541, in _call_impl
-[default7]:[rank7]:     return forward_call(*args, **kwargs)
-[default7]:[rank7]:   File "/fsx/ferdinandmom/ferdinand-hf/bench_cluster/nanotron/src/nanotron/parallel/pipeline_parallel/block.py", line 151, in forward
-[default7]:[rank7]:     output = self.pp_block(**new_kwargs)
-[default7]:[rank7]:   File "/fsx/ferdinandmom/miniforge3/envs/env-bench-cluster/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1532, in _wrapped_call_impl
-[default7]:[rank7]:     return self._call_impl(*args, **kwargs)
-[default7]:[rank7]:   File "/fsx/ferdinandmom/miniforge3/envs/env-bench-cluster/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1541, in _call_impl
-[default7]:[rank7]:     return forward_call(*args, **kwargs)
-[default7]:[rank7]:   File "/fsx/ferdinandmom/ferdinand-hf/bench_cluster/nanotron/src/nanotron/models/llama.py", line 629, in forward
-[default7]:[rank7]:     hidden_states = self.input_layernorm(hidden_states)
-[default7]:[rank7]:   File "/fsx/ferdinandmom/miniforge3/envs/env-bench-cluster/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1532, in _wrapped_call_impl
-[default7]:[rank7]:     return self._call_impl(*args, **kwargs)
-[default7]:[rank7]:   File "/fsx/ferdinandmom/miniforge3/envs/env-bench-cluster/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1541, in _call_impl
-[default7]:[rank7]:     return forward_call(*args, **kwargs)
-[default7]:[rank7]:   File "/fsx/ferdinandmom/ferdinand-hf/bench_cluster/nanotron/src/nanotron/nn/layer_norm.py", line 42, in forward
-[default7]:[rank7]:     return layer_norm_fn(
-[default7]:[rank7]:   File "/fsx/ferdinandmom/miniforge3/envs/env-bench-cluster/lib/python3.10/site-packages/flash_attn/ops/triton/layer_norm.py", line 875, in layer_norm_fn
-[default7]:[rank7]:     return LayerNormFn.apply(
-[default7]:[rank7]:   File "/fsx/ferdinandmom/miniforge3/envs/env-bench-cluster/lib/python3.10/site-packages/torch/autograd/function.py", line 598, in apply
-[default7]:[rank7]:     return super().apply(*args, **kwargs)  # type: ignore[misc]
-[default7]:[rank7]:   File "/fsx/ferdinandmom/miniforge3/envs/env-bench-cluster/lib/python3.10/site-packages/flash_attn/ops/triton/layer_norm.py", line 748, in forward
-[default7]:[rank7]:     y, y1, mean, rstd, residual_out, seeds, dropout_mask, dropout_mask1 = _layer_norm_fwd(
-[default7]:[rank7]:   File "/fsx/ferdinandmom/miniforge3/envs/env-bench-cluster/lib/python3.10/site-packages/flash_attn/ops/triton/layer_norm.py", line 335, in _layer_norm_fwd
-[default7]:[rank7]:     _layer_norm_fwd_1pass_kernel[(M,)](
-[default7]:[rank7]:   File "/fsx/ferdinandmom/miniforge3/envs/env-bench-cluster/lib/python3.10/site-packages/triton/runtime/jit.py", line 167, in <lambda>
-[default7]:[rank7]:     return lambda *args, **kwargs: self.run(grid=grid, warmup=False, *args, **kwargs)
-[default7]:[rank7]:   File "/fsx/ferdinandmom/miniforge3/envs/env-bench-cluster/lib/python3.10/site-packages/triton/runtime/autotuner.py", line 143, in run
-[default7]:[rank7]:     timings = {config: self._bench(*args, config=config, **kwargs) for config in pruned_configs}
-[default7]:[rank7]:   File "/fsx/ferdinandmom/miniforge3/envs/env-bench-cluster/lib/python3.10/site-packages/triton/runtime/autotuner.py", line 143, in <dictcomp>
-[default7]:[rank7]:     timings = {config: self._bench(*args, config=config, **kwargs) for config in pruned_configs}
-[default7]:[rank7]:   File "/fsx/ferdinandmom/miniforge3/envs/env-bench-cluster/lib/python3.10/site-packages/triton/runtime/autotuner.py", line 122, in _bench
-[default7]:[rank7]:     return do_bench(kernel_call, warmup=self.warmup, rep=self.rep, quantiles=(0.5, 0.2, 0.8))
-[default7]:[rank7]:   File "/fsx/ferdinandmom/miniforge3/envs/env-bench-cluster/lib/python3.10/site-packages/triton/testing.py", line 102, in do_bench
-[default7]:[rank7]:     fn()
-[default7]:[rank7]:   File "/fsx/ferdinandmom/miniforge3/envs/env-bench-cluster/lib/python3.10/site-packages/triton/runtime/autotuner.py", line 110, in kernel_call
-[default7]:[rank7]:     self.fn.run(
-[default7]:[rank7]:   File "/fsx/ferdinandmom/miniforge3/envs/env-bench-cluster/lib/python3.10/site-packages/triton/runtime/autotuner.py", line 305, in run
-[default7]:[rank7]:     return self.fn.run(*args, **kwargs)
-[default7]:[rank7]:   File "/fsx/ferdinandmom/miniforge3/envs/env-bench-cluster/lib/python3.10/site-packages/triton/runtime/autotuner.py", line 305, in run
-[default7]:[rank7]:     return self.fn.run(*args, **kwargs)
-[default7]:[rank7]:   File "/fsx/ferdinandmom/miniforge3/envs/env-bench-cluster/lib/python3.10/site-packages/triton/runtime/autotuner.py", line 305, in run
-[default7]:[rank7]:     return self.fn.run(*args, **kwargs)
-[default7]:[rank7]:   File "/fsx/ferdinandmom/miniforge3/envs/env-bench-cluster/lib/python3.10/site-packages/triton/runtime/jit.py", line 416, in run
-[default7]:[rank7]:     self.cache[device][key] = compile(
-[default7]:[rank7]:   File "/fsx/ferdinandmom/miniforge3/envs/env-bench-cluster/lib/python3.10/site-packages/triton/compiler/compiler.py", line 194, in compile
-[default7]:[rank7]:     metadata_group[f"{src.name}.{ext}"] = fn_cache_manager.put(next_module, f"{src.name}.{ext}")
-[default7]:[rank7]:   File "/fsx/ferdinandmom/miniforge3/envs/env-bench-cluster/lib/python3.10/site-packages/triton/runtime/cache.py", line 123, in put
-[default7]:[rank7]:     with open(temp_path, mode) as f:
-[default7]:[rank7]: OSError: [Errno 122] Disk quota exceeded
-[default4]:[rank4]: OSError: [Errno 122] Disk quota exceeded
-[default4]:
-[default4]:[rank4]: During handling of the above exception, another exception occurred:
-[default4]:
-[default4]:[rank4]: Traceback (most recent call last):
-[default4]:[rank4]:   File "/fsx/ferdinandmom/ferdinand-hf/bench_cluster/nanotron/run_train.py", line 237, in <module>
-[default4]:[rank4]:     trainer.train(dataloader)
-[default4]:[rank4]:   File "/fsx/ferdinandmom/ferdinand-hf/bench_cluster/nanotron/src/nanotron/trainer.py", line 429, in train
-[default4]:[rank4]:     outputs, loss_avg = self.training_step(dataloader=self.current_dataloader)
-[default4]:[rank4]:   File "/fsx/ferdinandmom/ferdinand-hf/bench_cluster/nanotron/src/nanotron/trainer.py", line 462, in training_step
-[default4]:[rank4]:     outputs = self.pipeline_engine.train_batch_iter(
-[default4]:[rank4]:   File "/fsx/ferdinandmom/ferdinand-hf/bench_cluster/nanotron/src/nanotron/parallel/pipeline_parallel/engine.py", line 278, in train_batch_iter
-[default4]:[rank4]:     output = self.forward(context=context, state=state, micro_batch=micro_batch, model=model)
-[default4]:[rank4]:   File "/fsx/ferdinandmom/ferdinand-hf/bench_cluster/nanotron/src/nanotron/parallel/pipeline_parallel/engine.py", line 44, in forward
-[default4]:[rank4]:     output = model(**micro_batch)
-[default4]:[rank4]:   File "/fsx/ferdinandmom/miniforge3/envs/env-bench-cluster/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1532, in _wrapped_call_impl
-[default4]:[rank4]:     return self._call_impl(*args, **kwargs)
-[default4]:[rank4]:   File "/fsx/ferdinandmom/miniforge3/envs/env-bench-cluster/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1541, in _call_impl
-[default4]:[rank4]:     return forward_call(*args, **kwargs)
-[default4]:[rank4]:   File "/fsx/ferdinandmom/ferdinand-hf/bench_cluster/nanotron/src/nanotron/models/llama.py", line 891, in forward
-[default4]:[rank4]:     sharded_logits = self.model(
-[default4]:[rank4]:   File "/fsx/ferdinandmom/miniforge3/envs/env-bench-cluster/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1532, in _wrapped_call_impl
-[default4]:[rank4]:     return self._call_impl(*args, **kwargs)
-[default4]:[rank4]:   File "/fsx/ferdinandmom/miniforge3/envs/env-bench-cluster/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1541, in _call_impl
-[default4]:[rank4]:     return forward_call(*args, **kwargs)
-[default4]:[rank4]:   File "/fsx/ferdinandmom/ferdinand-hf/bench_cluster/nanotron/src/nanotron/models/llama.py", line 764, in forward
-[default4]:[rank4]:     return self.forward_with_hidden_states(input_ids=input_ids, input_mask=input_mask)[0]
-[default4]:[rank4]:   File "/fsx/ferdinandmom/ferdinand-hf/bench_cluster/nanotron/src/nanotron/models/llama.py", line 780, in forward_with_hidden_states
-[default4]:[rank4]:     hidden_encoder_states = encoder_block(**hidden_encoder_states)
-[default4]:[rank4]:   File "/fsx/ferdinandmom/miniforge3/envs/env-bench-cluster/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1532, in _wrapped_call_impl
-[default4]:[rank4]:     return self._call_impl(*args, **kwargs)
-[default4]:[rank4]:   File "/fsx/ferdinandmom/miniforge3/envs/env-bench-cluster/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1541, in _call_impl
-[default4]:[rank4]:     return forward_call(*args, **kwargs)
-[default4]:[rank4]:   File "/fsx/ferdinandmom/ferdinand-hf/bench_cluster/nanotron/src/nanotron/parallel/pipeline_parallel/block.py", line 151, in forward
-[default4]:[rank4]:     output = self.pp_block(**new_kwargs)
-[default4]:[rank4]:   File "/fsx/ferdinandmom/miniforge3/envs/env-bench-cluster/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1532, in _wrapped_call_impl
-[default4]:[rank4]:     return self._call_impl(*args, **kwargs)
-[default4]:[rank4]:   File "/fsx/ferdinandmom/miniforge3/envs/env-bench-cluster/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1541, in _call_impl
-[default4]:[rank4]:     return forward_call(*args, **kwargs)
-[default4]:[rank4]:   File "/fsx/ferdinandmom/ferdinand-hf/bench_cluster/nanotron/src/nanotron/models/llama.py", line 629, in forward
-[default4]:[rank4]:     hidden_states = self.input_layernorm(hidden_states)
-[default4]:[rank4]:   File "/fsx/ferdinandmom/miniforge3/envs/env-bench-cluster/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1532, in _wrapped_call_impl
-[default4]:[rank4]:     return self._call_impl(*args, **kwargs)
-[default4]:[rank4]:   File "/fsx/ferdinandmom/miniforge3/envs/env-bench-cluster/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1541, in _call_impl
-[default4]:[rank4]:     return forward_call(*args, **kwargs)
-[default4]:[rank4]:   File "/fsx/ferdinandmom/ferdinand-hf/bench_cluster/nanotron/src/nanotron/nn/layer_norm.py", line 42, in forward
-[default4]:[rank4]:     return layer_norm_fn(
-[default4]:[rank4]:   File "/fsx/ferdinandmom/miniforge3/envs/env-bench-cluster/lib/python3.10/site-packages/flash_attn/ops/triton/layer_norm.py", line 875, in layer_norm_fn
-[default4]:[rank4]:     return LayerNormFn.apply(
-[default4]:[rank4]:   File "/fsx/ferdinandmom/miniforge3/envs/env-bench-cluster/lib/python3.10/site-packages/torch/autograd/function.py", line 598, in apply
-[default4]:[rank4]:     return super().apply(*args, **kwargs)  # type: ignore[misc]
-[default4]:[rank4]:   File "/fsx/ferdinandmom/miniforge3/envs/env-bench-cluster/lib/python3.10/site-packages/flash_attn/ops/triton/layer_norm.py", line 748, in forward
-[default4]:[rank4]:     y, y1, mean, rstd, residual_out, seeds, dropout_mask, dropout_mask1 = _layer_norm_fwd(
-[default4]:[rank4]:   File "/fsx/ferdinandmom/miniforge3/envs/env-bench-cluster/lib/python3.10/site-packages/flash_attn/ops/triton/layer_norm.py", line 335, in _layer_norm_fwd
-[default4]:[rank4]:     _layer_norm_fwd_1pass_kernel[(M,)](
-[default4]:[rank4]:   File "/fsx/ferdinandmom/miniforge3/envs/env-bench-cluster/lib/python3.10/site-packages/triton/runtime/jit.py", line 167, in <lambda>
-[default4]:[rank4]:     return lambda *args, **kwargs: self.run(grid=grid, warmup=False, *args, **kwargs)
-[default4]:[rank4]:   File "/fsx/ferdinandmom/miniforge3/envs/env-bench-cluster/lib/python3.10/site-packages/triton/runtime/autotuner.py", line 143, in run
-[default4]:[rank4]:     timings = {config: self._bench(*args, config=config, **kwargs) for config in pruned_configs}
-[default4]:[rank4]:   File "/fsx/ferdinandmom/miniforge3/envs/env-bench-cluster/lib/python3.10/site-packages/triton/runtime/autotuner.py", line 143, in <dictcomp>
-[default4]:[rank4]:     timings = {config: self._bench(*args, config=config, **kwargs) for config in pruned_configs}
-[default4]:[rank4]:   File "/fsx/ferdinandmom/miniforge3/envs/env-bench-cluster/lib/python3.10/site-packages/triton/runtime/autotuner.py", line 122, in _bench
-[default4]:[rank4]:     return do_bench(kernel_call, warmup=self.warmup, rep=self.rep, quantiles=(0.5, 0.2, 0.8))
-[default4]:[rank4]:   File "/fsx/ferdinandmom/miniforge3/envs/env-bench-cluster/lib/python3.10/site-packages/triton/testing.py", line 102, in do_bench
-[default4]:[rank4]:     fn()
-[default4]:[rank4]:   File "/fsx/ferdinandmom/miniforge3/envs/env-bench-cluster/lib/python3.10/site-packages/triton/runtime/autotuner.py", line 110, in kernel_call
-[default4]:[rank4]:     self.fn.run(
-[default4]:[rank4]:   File "/fsx/ferdinandmom/miniforge3/envs/env-bench-cluster/lib/python3.10/site-packages/triton/runtime/autotuner.py", line 305, in run
-[default4]:[rank4]:     return self.fn.run(*args, **kwargs)
-[default4]:[rank4]:   File "/fsx/ferdinandmom/miniforge3/envs/env-bench-cluster/lib/python3.10/site-packages/triton/runtime/autotuner.py", line 305, in run
-[default4]:[rank4]:     return self.fn.run(*args, **kwargs)
-[default4]:[rank4]:   File "/fsx/ferdinandmom/miniforge3/envs/env-bench-cluster/lib/python3.10/site-packages/triton/runtime/autotuner.py", line 305, in run
-[default4]:[rank4]:     return self.fn.run(*args, **kwargs)
-[default4]:[rank4]:   File "/fsx/ferdinandmom/miniforge3/envs/env-bench-cluster/lib/python3.10/site-packages/triton/runtime/jit.py", line 416, in run
-[default4]:[rank4]:     self.cache[device][key] = compile(
-[default4]:[rank4]:   File "/fsx/ferdinandmom/miniforge3/envs/env-bench-cluster/lib/python3.10/site-packages/triton/compiler/compiler.py", line 194, in compile
-[default4]:[rank4]:     metadata_group[f"{src.name}.{ext}"] = fn_cache_manager.put(next_module, f"{src.name}.{ext}")
-[default4]:[rank4]:   File "/fsx/ferdinandmom/miniforge3/envs/env-bench-cluster/lib/python3.10/site-packages/triton/runtime/cache.py", line 123, in put
-[default4]:[rank4]:     with open(temp_path, mode) as f:
-[default4]:[rank4]: OSError: [Errno 122] Disk quota exceeded
-[default4]:[rank12]: OSError: [Errno 122] Disk quota exceeded
-[default4]:
-[default4]:[rank12]: During handling of the above exception, another exception occurred:
-[default4]:
-[default4]:[rank12]: Traceback (most recent call last):
-[default4]:[rank12]:   File "/fsx/ferdinandmom/ferdinand-hf/bench_cluster/nanotron/run_train.py", line 237, in <module>
-[default4]:[rank12]:     trainer.train(dataloader)
-[default4]:[rank12]:   File "/fsx/ferdinandmom/ferdinand-hf/bench_cluster/nanotron/src/nanotron/trainer.py", line 429, in train
-[default4]:[rank12]:     outputs, loss_avg = self.training_step(dataloader=self.current_dataloader)
-[default4]:[rank12]:   File "/fsx/ferdinandmom/ferdinand-hf/bench_cluster/nanotron/src/nanotron/trainer.py", line 462, in training_step
-[default4]:[rank12]:     outputs = self.pipeline_engine.train_batch_iter(
-[default4]:[rank12]:   File "/fsx/ferdinandmom/ferdinand-hf/bench_cluster/nanotron/src/nanotron/parallel/pipeline_parallel/engine.py", line 278, in train_batch_iter
-[default4]:[rank12]:     output = self.forward(context=context, state=state, micro_batch=micro_batch, model=model)
-[default4]:[rank12]:   File "/fsx/ferdinandmom/ferdinand-hf/bench_cluster/nanotron/src/nanotron/parallel/pipeline_parallel/engine.py", line 44, in forward
-[default4]:[rank12]:     output = model(**micro_batch)
-[default4]:[rank12]:   File "/fsx/ferdinandmom/miniforge3/envs/env-bench-cluster/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1532, in _wrapped_call_impl
-[default4]:[rank12]:     return self._call_impl(*args, **kwargs)
-[default4]:[rank12]:   File "/fsx/ferdinandmom/miniforge3/envs/env-bench-cluster/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1541, in _call_impl
-[default4]:[rank12]:     return forward_call(*args, **kwargs)
-[default4]:[rank12]:   File "/fsx/ferdinandmom/ferdinand-hf/bench_cluster/nanotron/src/nanotron/models/llama.py", line 891, in forward
-[default4]:[rank12]:     sharded_logits = self.model(
-[default4]:[rank12]:   File "/fsx/ferdinandmom/miniforge3/envs/env-bench-cluster/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1532, in _wrapped_call_impl
-[default4]:[rank12]:     return self._call_impl(*args, **kwargs)
-[default4]:[rank12]:   File "/fsx/ferdinandmom/miniforge3/envs/env-bench-cluster/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1541, in _call_impl
-[default4]:[rank12]:     return forward_call(*args, **kwargs)
-[default4]:[rank12]:   File "/fsx/ferdinandmom/ferdinand-hf/bench_cluster/nanotron/src/nanotron/models/llama.py", line 764, in forward
-[default4]:[rank12]:     return self.forward_with_hidden_states(input_ids=input_ids, input_mask=input_mask)[0]
-[default4]:[rank12]:   File "/fsx/ferdinandmom/ferdinand-hf/bench_cluster/nanotron/src/nanotron/models/llama.py", line 780, in forward_with_hidden_states
-[default4]:[rank12]:     hidden_encoder_states = encoder_block(**hidden_encoder_states)
-[default4]:[rank12]:   File "/fsx/ferdinandmom/miniforge3/envs/env-bench-cluster/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1532, in _wrapped_call_impl
-[default4]:[rank12]:     return self._call_impl(*args, **kwargs)
-[default4]:[rank12]:   File "/fsx/ferdinandmom/miniforge3/envs/env-bench-cluster/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1541, in _call_impl
-[default4]:[rank12]:     return forward_call(*args, **kwargs)
-[default4]:[rank12]:   File "/fsx/ferdinandmom/ferdinand-hf/bench_cluster/nanotron/src/nanotron/parallel/pipeline_parallel/block.py", line 151, in forward
-[default4]:[rank12]:     output = self.pp_block(**new_kwargs)
-[default4]:[rank12]:   File "/fsx/ferdinandmom/miniforge3/envs/env-bench-cluster/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1532, in _wrapped_call_impl
-[default4]:[rank12]:     return self._call_impl(*args, **kwargs)
-[default4]:[rank12]:   File "/fsx/ferdinandmom/miniforge3/envs/env-bench-cluster/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1541, in _call_impl
-[default4]:[rank12]:     return forward_call(*args, **kwargs)
-[default4]:[rank12]:   File "/fsx/ferdinandmom/ferdinand-hf/bench_cluster/nanotron/src/nanotron/models/llama.py", line 629, in forward
-[default4]:[rank12]:     hidden_states = self.input_layernorm(hidden_states)
-[default4]:[rank12]:   File "/fsx/ferdinandmom/miniforge3/envs/env-bench-cluster/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1532, in _wrapped_call_impl
-[default4]:[rank12]:     return self._call_impl(*args, **kwargs)
-[default4]:[rank12]:   File "/fsx/ferdinandmom/miniforge3/envs/env-bench-cluster/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1541, in _call_impl
-[default4]:[rank12]:     return forward_call(*args, **kwargs)
-[default4]:[rank12]:   File "/fsx/ferdinandmom/ferdinand-hf/bench_cluster/nanotron/src/nanotron/nn/layer_norm.py", line 42, in forward
-[default4]:[rank12]:     return layer_norm_fn(
-[default4]:[rank12]:   File "/fsx/ferdinandmom/miniforge3/envs/env-bench-cluster/lib/python3.10/site-packages/flash_attn/ops/triton/layer_norm.py", line 875, in layer_norm_fn
-[default4]:[rank12]:     return LayerNormFn.apply(
-[default4]:[rank12]:   File "/fsx/ferdinandmom/miniforge3/envs/env-bench-cluster/lib/python3.10/site-packages/torch/autograd/function.py", line 598, in apply
-[default4]:[rank12]:     return super().apply(*args, **kwargs)  # type: ignore[misc]
-[default4]:[rank12]:   File "/fsx/ferdinandmom/miniforge3/envs/env-bench-cluster/lib/python3.10/site-packages/flash_attn/ops/triton/layer_norm.py", line 748, in forward
-[default4]:[rank12]:     y, y1, mean, rstd, residual_out, seeds, dropout_mask, dropout_mask1 = _layer_norm_fwd(
-[default4]:[rank12]:   File "/fsx/ferdinandmom/miniforge3/envs/env-bench-cluster/lib/python3.10/site-packages/flash_attn/ops/triton/layer_norm.py", line 335, in _layer_norm_fwd
-[default4]:[rank12]:     _layer_norm_fwd_1pass_kernel[(M,)](
-[default4]:[rank12]:   File "/fsx/ferdinandmom/miniforge3/envs/env-bench-cluster/lib/python3.10/site-packages/triton/runtime/jit.py", line 167, in <lambda>
-[default4]:[rank12]:     return lambda *args, **kwargs: self.run(grid=grid, warmup=False, *args, **kwargs)
-[default4]:[rank12]:   File "/fsx/ferdinandmom/miniforge3/envs/env-bench-cluster/lib/python3.10/site-packages/triton/runtime/autotuner.py", line 143, in run
-[default4]:[rank12]:     timings = {config: self._bench(*args, config=config, **kwargs) for config in pruned_configs}
-[default4]:[rank12]:   File "/fsx/ferdinandmom/miniforge3/envs/env-bench-cluster/lib/python3.10/site-packages/triton/runtime/autotuner.py", line 143, in <dictcomp>
-[default4]:[rank12]:     timings = {config: self._bench(*args, config=config, **kwargs) for config in pruned_configs}
-[default4]:[rank12]:   File "/fsx/ferdinandmom/miniforge3/envs/env-bench-cluster/lib/python3.10/site-packages/triton/runtime/autotuner.py", line 122, in _bench
-[default4]:[rank12]:     return do_bench(kernel_call, warmup=self.warmup, rep=self.rep, quantiles=(0.5, 0.2, 0.8))
-[default4]:[rank12]:   File "/fsx/ferdinandmom/miniforge3/envs/env-bench-cluster/lib/python3.10/site-packages/triton/testing.py", line 102, in do_bench
-[default4]:[rank12]:     fn()
-[default4]:[rank12]:   File "/fsx/ferdinandmom/miniforge3/envs/env-bench-cluster/lib/python3.10/site-packages/triton/runtime/autotuner.py", line 110, in kernel_call
-[default4]:[rank12]:     self.fn.run(
-[default4]:[rank12]:   File "/fsx/ferdinandmom/miniforge3/envs/env-bench-cluster/lib/python3.10/site-packages/triton/runtime/autotuner.py", line 305, in run
-[default4]:[rank12]:     return self.fn.run(*args, **kwargs)
-[default4]:[rank12]:   File "/fsx/ferdinandmom/miniforge3/envs/env-bench-cluster/lib/python3.10/site-packages/triton/runtime/autotuner.py", line 305, in run
-[default4]:[rank12]:     return self.fn.run(*args, **kwargs)
-[default4]:[rank12]:   File "/fsx/ferdinandmom/miniforge3/envs/env-bench-cluster/lib/python3.10/site-packages/triton/runtime/autotuner.py", line 305, in run
-[default4]:[rank12]:     return self.fn.run(*args, **kwargs)
-[default4]:[rank12]:   File "/fsx/ferdinandmom/miniforge3/envs/env-bench-cluster/lib/python3.10/site-packages/triton/runtime/jit.py", line 416, in run
-[default4]:[rank12]:     self.cache[device][key] = compile(
-[default4]:[rank12]:   File "/fsx/ferdinandmom/miniforge3/envs/env-bench-cluster/lib/python3.10/site-packages/triton/compiler/compiler.py", line 194, in compile
-[default4]:[rank12]:     metadata_group[f"{src.name}.{ext}"] = fn_cache_manager.put(next_module, f"{src.name}.{ext}")
-[default4]:[rank12]:   File "/fsx/ferdinandmom/miniforge3/envs/env-bench-cluster/lib/python3.10/site-packages/triton/runtime/cache.py", line 123, in put
-[default4]:[rank12]:     with open(temp_path, mode) as f:
-[default4]:[rank12]: OSError: [Errno 122] Disk quota exceeded
-W0702 16:32:48.847000 140386122716992 torch/distributed/elastic/multiprocessing/api.py:851] Sending process 2427255 closing signal SIGTERM
-W0702 16:32:48.852000 140386122716992 torch/distributed/elastic/multiprocessing/api.py:851] Sending process 2427256 closing signal SIGTERM
-W0702 16:32:48.854000 140386122716992 torch/distributed/elastic/multiprocessing/api.py:851] Sending process 2427257 closing signal SIGTERM
-W0702 16:32:48.862000 139796830594880 torch/distributed/elastic/multiprocessing/api.py:851] Sending process 99623 closing signal SIGTERM
-W0702 16:32:48.863000 140386122716992 torch/distributed/elastic/multiprocessing/api.py:851] Sending process 2427258 closing signal SIGTERM
-W0702 16:32:48.866000 139796830594880 torch/distributed/elastic/multiprocessing/api.py:851] Sending process 99624 closing signal SIGTERM
-W0702 16:32:48.871000 139796830594880 torch/distributed/elastic/multiprocessing/api.py:851] Sending process 99625 closing signal SIGTERM
-W0702 16:32:48.875000 139796830594880 torch/distributed/elastic/multiprocessing/api.py:851] Sending process 99626 closing signal SIGTERM
-W0702 16:32:48.878000 139796830594880 torch/distributed/elastic/multiprocessing/api.py:851] Sending process 99628 closing signal SIGTERM
-W0702 16:32:48.880000 139796830594880 torch/distributed/elastic/multiprocessing/api.py:851] Sending process 99629 closing signal SIGTERM
-W0702 16:32:48.882000 139796830594880 torch/distributed/elastic/multiprocessing/api.py:851] Sending process 99630 closing signal SIGTERM
-W0702 16:32:48.891000 140386122716992 torch/distributed/elastic/multiprocessing/api.py:851] Sending process 2427260 closing signal SIGTERM
-W0702 16:32:48.894000 140386122716992 torch/distributed/elastic/multiprocessing/api.py:851] Sending process 2427261 closing signal SIGTERM
-E0702 16:32:51.295000 140386122716992 torch/distributed/elastic/multiprocessing/api.py:826] failed (exitcode: 1) local_rank: 4 (pid: 2427259) of binary: /fsx/ferdinandmom/miniforge3/envs/env-bench-cluster/bin/python3.10
-Traceback (most recent call last):
-  File "/fsx/ferdinandmom/miniforge3/envs/env-bench-cluster/bin/torchrun", line 8, in <module>
-    sys.exit(main())
-  File "/fsx/ferdinandmom/miniforge3/envs/env-bench-cluster/lib/python3.10/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 347, in wrapper
-    return f(*args, **kwargs)
-  File "/fsx/ferdinandmom/miniforge3/envs/env-bench-cluster/lib/python3.10/site-packages/torch/distributed/run.py", line 879, in main
-    run(args)
-  File "/fsx/ferdinandmom/miniforge3/envs/env-bench-cluster/lib/python3.10/site-packages/torch/distributed/run.py", line 870, in run
-    elastic_launch(
-  File "/fsx/ferdinandmom/miniforge3/envs/env-bench-cluster/lib/python3.10/site-packages/torch/distributed/launcher/api.py", line 132, in __call__
-    return launch_agent(self._config, self._entrypoint, list(args))
-  File "/fsx/ferdinandmom/miniforge3/envs/env-bench-cluster/lib/python3.10/site-packages/torch/distributed/launcher/api.py", line 263, in launch_agent
-    raise ChildFailedError(
-torch.distributed.elastic.multiprocessing.errors.ChildFailedError: 
-============================================================
-/fsx/ferdinandmom/ferdinand-hf/bench_cluster/nanotron/run_train.py FAILED
-------------------------------------------------------------
-Failures:
-[1]:
-  time      : 2024-07-02_16:32:48
-  host      : ip-26-0-169-239.ec2.internal
-  rank      : 7 (local_rank: 7)
-  exitcode  : 1 (pid: 2427262)
-  error_file: <N/A>
-  traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html
-------------------------------------------------------------
-Root Cause (first observed failure):
-[0]:
-  time      : 2024-07-02_16:32:48
-  host      : ip-26-0-169-239.ec2.internal
-  rank      : 4 (local_rank: 4)
-  exitcode  : 1 (pid: 2427259)
-  error_file: <N/A>
-  traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html
-============================================================
-E0702 16:32:51.480000 139796830594880 torch/distributed/elastic/multiprocessing/api.py:826] failed (exitcode: 1) local_rank: 4 (pid: 99627) of binary: /fsx/ferdinandmom/miniforge3/envs/env-bench-cluster/bin/python3.10
-W0702 16:32:51.487000 139796830594880 torch/distributed/elastic/rendezvous/dynamic_rendezvous.py:1203] The node 'ip-26-0-169-247.ec2.internal_99554_0' has failed to shutdown the rendezvous 'none' due to an error of type RendezvousConnectionError.
-W0702 16:32:51.515000 139796830594880 torch/distributed/elastic/rendezvous/dynamic_rendezvous.py:1203] The node 'ip-26-0-169-247.ec2.internal_99554_0' has failed to shutdown the rendezvous 'none' due to an error of type RendezvousConnectionError.
-W0702 16:32:51.525000 139796830594880 torch/distributed/elastic/rendezvous/dynamic_rendezvous.py:1203] The node 'ip-26-0-169-247.ec2.internal_99554_0' has failed to shutdown the rendezvous 'none' due to an error of type RendezvousConnectionError.
-Traceback (most recent call last):
-  File "/fsx/ferdinandmom/miniforge3/envs/env-bench-cluster/bin/torchrun", line 8, in <module>
-    sys.exit(main())
-  File "/fsx/ferdinandmom/miniforge3/envs/env-bench-cluster/lib/python3.10/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 347, in wrapper
-    return f(*args, **kwargs)
-  File "/fsx/ferdinandmom/miniforge3/envs/env-bench-cluster/lib/python3.10/site-packages/torch/distributed/run.py", line 879, in main
-    run(args)
-  File "/fsx/ferdinandmom/miniforge3/envs/env-bench-cluster/lib/python3.10/site-packages/torch/distributed/run.py", line 870, in run
-    elastic_launch(
-  File "/fsx/ferdinandmom/miniforge3/envs/env-bench-cluster/lib/python3.10/site-packages/torch/distributed/launcher/api.py", line 132, in __call__
-    return launch_agent(self._config, self._entrypoint, list(args))
-  File "/fsx/ferdinandmom/miniforge3/envs/env-bench-cluster/lib/python3.10/site-packages/torch/distributed/launcher/api.py", line 263, in launch_agent
-    raise ChildFailedError(
-torch.distributed.elastic.multiprocessing.errors.ChildFailedError: 
-============================================================
-/fsx/ferdinandmom/ferdinand-hf/bench_cluster/nanotron/run_train.py FAILED
-------------------------------------------------------------
-Failures:
-  <NO_OTHER_FAILURES>
-------------------------------------------------------------
-Root Cause (first observed failure):
-[0]:
-  time      : 2024-07-02_16:32:48
-  host      : ip-26-0-169-247.ec2.internal
-  rank      : 12 (local_rank: 4)
-  exitcode  : 1 (pid: 99627)
-  error_file: <N/A>
-  traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html
-============================================================
-srun: error: ip-26-0-169-239: task 0: Exited with exit code 1
-srun: error: ip-26-0-169-247: task 1: Exited with exit code 1
-Consider using `hf_transfer` for faster uploads. This solution comes with some limitations. See https://huggingface.co/docs/huggingface_hub/hf_transfer for more details.

llama-1B/16_GPUS/dp-1_tp-16_pp-1_mbz-32/status.txt

@@ -1 +0,0 @@
-fail

llama-1B/16_GPUS/dp-1_tp-16_pp-1_mbz-64/bench.slurm

@@ -1,111 +0,0 @@
-#!/bin/bash
-
-#SBATCH --job-name=bench_cluster
-#SBATCH --time=00:59:00
-#SBATCH --partition=hopper-prod
-#SBATCH --nodes=2
-#SBATCH --gres=gpu:8
-#SBATCH --qos=high
-#SBATCH --ntasks-per-node=1
-#SBATCH --cpus-per-task=96
-#SBATCH --exclusive
-#SBATCH --output=/fsx/ferdinandmom/ferdinand-hf/bench_cluster/results/llama-1B/16_GPUS/dp-1_tp-16_pp-1_mbz-64/log.out
-#SBATCH --error=/fsx/ferdinandmom/ferdinand-hf/bench_cluster/results/llama-1B/16_GPUS/dp-1_tp-16_pp-1_mbz-64/log.out
-
-# Function to update status based on squeue output
-update_status() {
-    job_id=$1
-    status_file=$2
-    # For unknown reasons, it doenst update status for pending. It only works for running 
-    while true; do
-        job_status=$(squeue --job $job_id --noheader --format=%T)
-        echo "Job status: $job_status"
-        if [ -z "$job_status" ]; then
-            # Job has finished or is not found
-            break
-        elif [ "$job_status" = "RUNNING" ]; then
-            printf "running" > $status_file
-            break
-        fi
-        sleep 10
-    done
-}
-
-# Misc initializations.
-echo "========================"
-echo "START TIME: $(date)"
-source /fsx/ferdinandmom/miniforge3/etc/profile.d/conda.sh
-conda activate /fsx/ferdinandmom/miniforge3/envs/env-bench-cluster
-echo python3 version = $(python3 --version)
-echo "========================"
-
-# Slurm stuff
-export HOSTNAMES=$(scontrol show hostnames "$SLURM_JOB_NODELIST")
-export MASTER_ADDR=$(scontrol show hostnames "$SLURM_JOB_NODELIST" | head -n 1)
-export MASTER_PORT=$((1024 + RANDOM % 64511))
-
-export TMPDIR=/scratch
-export HF_DATASETS_CACHE="/admin/home/ferdinand_mom/.cache"
-export CUBLAS_WORKSPACE_CONFIG=":4096:8"
-export CUDA_DEVICE_MAX_CONNECTIONS="1"
-
-huggingface-cli login --token $HUGGINGFACE_TOKEN
-
-
-NANOTRON_REPO="/fsx/ferdinandmom/ferdinand-hf/bench_cluster/nanotron"
-CMD="$NANOTRON_REPO/run_train.py --config-file /fsx/ferdinandmom/ferdinand-hf/bench_cluster/results/llama-1B/16_GPUS/dp-1_tp-16_pp-1_mbz-64/config.yaml"
-
-LAUNCHER="torchrun \
-   --nproc_per_node 8 \
-   --nnodes 2 \
-   --rdzv_endpoint ${MASTER_ADDR}:${MASTER_PORT} \
-   --rdzv_backend c10d \
-   --max_restarts 0 \
-   --tee 3 \
-   --node_rank ${SLURM_PROCID}"
-
-# Checkout the bench_cluster branch
-cd $NANOTRON_REPO
-git checkout bench_cluster
-cd ..
-# Get the current job ID
-job_id=${SLURM_JOB_ID}
-
-# Update status to "pending" or "running" in the background
-update_status $job_id /fsx/ferdinandmom/ferdinand-hf/bench_cluster/results/llama-1B/16_GPUS/dp-1_tp-16_pp-1_mbz-64/status.txt &
-
-# Run the main command
-srun -u $LAUNCHER $CMD
-exit_status=$?
-
-# Update status based on the exit status of `srun`
-if [ $exit_status -eq 0 ]; then
-    printf "completed" > /fsx/ferdinandmom/ferdinand-hf/bench_cluster/results/llama-1B/16_GPUS/dp-1_tp-16_pp-1_mbz-64/status.txt
-else
-    if grep -q "OutOfMemoryError" /fsx/ferdinandmom/ferdinand-hf/bench_cluster/results/llama-1B/16_GPUS/dp-1_tp-16_pp-1_mbz-64/log.out; then
-        printf "oom" > /fsx/ferdinandmom/ferdinand-hf/bench_cluster/results/llama-1B/16_GPUS/dp-1_tp-16_pp-1_mbz-64/status.txt
-    elif grep -q " CUDA error: an illegal memory access" /fsx/ferdinandmom/ferdinand-hf/bench_cluster/results/llama-1B/16_GPUS/dp-1_tp-16_pp-1_mbz-64/log.out; then
-        printf "oom" > /fsx/ferdinandmom/ferdinand-hf/bench_cluster/results/llama-1B/16_GPUS/dp-1_tp-16_pp-1_mbz-64/status.txt
-    elif grep -q "Timeout at NCCL" /fsx/ferdinandmom/ferdinand-hf/bench_cluster/results/llama-1B/16_GPUS/dp-1_tp-16_pp-1_mbz-64/log.out; then
-        printf "timeout" > /fsx/ferdinandmom/ferdinand-hf/bench_cluster/results/llama-1B/16_GPUS/dp-1_tp-16_pp-1_mbz-64/status.txt
-    else
-        printf "fail" > /fsx/ferdinandmom/ferdinand-hf/bench_cluster/results/llama-1B/16_GPUS/dp-1_tp-16_pp-1_mbz-64/status.txt
-    fi 
-fi
-
-# Run the report script if the job completed successfully
-if [ $exit_status -eq 0 ]; then
-    python /fsx/ferdinandmom/ferdinand-hf/bench_cluster/main.py report --inp_dir /fsx/ferdinandmom/ferdinand-hf/bench_cluster/results/llama-1B/16_GPUS/dp-1_tp-16_pp-1_mbz-64 --is_logs
-    python /fsx/ferdinandmom/ferdinand-hf/bench_cluster/main.py report --inp_dir /fsx/ferdinandmom/ferdinand-hf/bench_cluster/results/llama-1B/16_GPUS/dp-1_tp-16_pp-1_mbz-64 --is_profiler
-fi
-
-
-# Push to hub the folder using huggingface_cli
-huggingface-cli upload nanotron/bench_cluster /fsx/ferdinandmom/ferdinand-hf/bench_cluster/results/llama-1B/16_GPUS/dp-1_tp-16_pp-1_mbz-64 llama-1B/16_GPUS/dp-1_tp-16_pp-1_mbz-64 --commit-message "Upload llama-1B/16_GPUS/dp-1_tp-16_pp-1_mbz-64"
-
-# Verify the upload
-if [ $? -eq 0 ]; then
-    echo "Uploading to Huggingface Hub successful"
-else
-    echo "Failed to upload to Huggingface Hub"
-fi

llama-1B/16_GPUS/dp-1_tp-16_pp-1_mbz-64/config.yaml

@@ -1,90 +0,0 @@
-general:
-  project: bench_cluster
-  seed: 42
-model:
-  ddp_bucket_cap_mb: 25
-  dtype: bfloat16
-  init_method:
-    std: 0.025
-  make_vocab_size_divisible_by: 1
-  model_config:
-    bos_token_id: 1
-    eos_token_id: 2
-    hidden_act: silu
-    hidden_size: 2048
-    initializer_range: 0.02
-    intermediate_size: 4096
-    is_llama_config: true
-    max_position_embeddings: 4096
-    num_attention_heads: 32
-    num_hidden_layers: 24
-    num_key_value_heads: 32
-    pad_token_id: null
-    pretraining_tp: 1
-    rms_norm_eps: 1.0e-05
-    rope_scaling: null
-    rope_theta: 10000.0
-    tie_word_embeddings: true
-    use_cache: true
-    vocab_size: 50257
-optimizer:
-  accumulate_grad_in_fp32: true
-  clip_grad: 1.0
-  learning_rate_scheduler:
-    learning_rate: 0.0001
-    lr_decay_style: linear
-    lr_warmup_style: linear
-    lr_warmup_steps: 1
-    min_decay_lr: 1.0e-05
-  optimizer_factory:
-    adam_beta1: 0.9
-    adam_beta2: 0.95
-    adam_eps: 1.0e-08
-    name: adamW
-    torch_adam_is_fused: true
-  weight_decay: 0.01
-  zero_stage: 1
-parallelism:
-  dp: 1
-  expert_parallel_size: 1
-  pp: 1
-  pp_engine: 1f1b
-  tp: 16
-  tp_linear_async_communication: false
-  tp_mode: REDUCE_SCATTER
-profiler:
-  profiler_export_path: /fsx/ferdinandmom/ferdinand-hf/bench_cluster/results/llama-1B/16_GPUS/dp-1_tp-16_pp-1_mbz-64
-tokenizer:
-  tokenizer_max_length: null
-  tokenizer_name_or_path: openai-community/gpt2
-  tokenizer_revision: null
-data_stages:
-- name: Training Stage
-  start_training_step: 1
-  data:
-    dataset:
-      dataset_overwrite_cache: false
-      dataset_processing_num_proc_per_process: 64
-      hf_dataset_config_name: null
-      hf_dataset_or_datasets: roneneldan/TinyStories
-      hf_dataset_splits: train
-      text_column_name: text
-    num_loading_workers: 32
-    seed: 42
-lighteval: null
-tokens:
-  train_steps: 20
-  val_check_interval: -1
-  batch_accumulation_per_replica: 16
-  limit_test_batches: 0
-  limit_val_batches: 0
-  micro_batch_size: 64
-  sequence_length: 4096
-logging:
-  iteration_step_info_interval: 1
-  log_level: info
-  log_level_replica: info
-checkpoints:
-  checkpoint_interval: 100000
-  checkpoints_path: /dev/null
-  resume_checkpoint_path: null

llama-1B/16_GPUS/dp-1_tp-16_pp-1_mbz-64/status.txt

@@ -1 +0,0 @@
-oom