p-doom/crowd-code-dataset-0.2 huggingface dataset:
[tab] experiments/tokenizer_time_measurement_grain.sh (shellscript)
#!/usr/bin/env bash
source .venv/bin/activate


data_dir='data_arrayrecord/dummy'

# FIXME: the `time_measurement_interval` line is only for debugging and should be removed
srun python train_tokenizer.py \
    --time_measurement_interval 10 \
    --batch_size 48 \
    --num_steps 300000 \
    --warmup_steps 10000 \
    --seed 0 \
    --min_lr=0.0000866 \
    --max_lr=0.0000866 \
    --data_dir $data_dir
[content] experiments/tokenizer_cross_node_checkpointing_test.sh (shellscript)
#!/usr/bin/env bash
source .venv/bin/activate


data_dir='data_arrayrecord/dummy'

# FIXME: the `time_measurement_interval` line is only for debugging and should be removed
srun python train_tokenizer.py \
    --time_measurement_interval 10 \
    --batch_size 48 \
    --num_steps 300000 \
    --warmup_steps 10000 \
    --seed 0 \
    --min_lr=0.0000866 \
    --max_lr=0.0000866 \
    --data_dir $data_dir