p-doom/crowd-code-dataset-0.2 huggingface dataset:
[tab] experiments/sbatch_tokenizer_dummy_distributed_checkpoint_test.sh (shellscript)
#!/bin/bash

#SBATCH --nodes=4
#SBATCH --ntasks-per-node=4
#SBATCH --gres=gpu:4
#SBATCH --cpus-per-task=8
#SBATCH --time=00:10:00
#SBATCH --partition=accelerated
#SBATCH --account=hk-project-p0023960
#SBATCH --job-name=test
module unload mpi/openmpi/5.0
module unload devel/cuda/12.4
source .venv/bin/activate


data_dir='/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/data/dummy'

srun python train_tokenizer.py \
    --batch_size 12 \
    --ckpt_dir checkpoints/tokenizer_dummy_distributed_checkpointing_test \
    --log_checkpoint_interval 10 \
    --num_steps 300000 \
    --warmup_steps 10000 \
    --seed 0 \
    --min_lr=0.0000866 \
    --max_lr=0.0000866 \
    --data_dir $data_dir