| python run_clm_mp.py \ | |
| --model_name_or_path /mnt/disks/flaxdisk/norwegian-gptneo-blue/ \ | |
| --tokenizer_name /mnt/disks/flaxdisk/norwegian-gptneo-blue/ \ | |
| --train_file /mnt/disks/flaxdisk/corpus/administrative_nb_train.json \ | |
| --validation_file /mnt/disks/flaxdisk/corpus/administrative_nb_validation.json \ | |
| --do_train \ | |
| --do_eval \ | |
| --block_size 1024 \ | |
| --num_train_epochs 5 \ | |
| --learning_rate 4e-6 \ | |
| --per_device_train_batch_size 2 \ | |
| --per_device_eval_batch_size 2 \ | |
| --overwrite_output_dir \ | |
| --output_dir "./" \ | |
| --cache_dir /mnt/disks/flaxdisk/cache/ \ | |
| --preprocessing_num_workers 96 \ | |
| --dtype bfloat16 \ | |
| --logging_steps 50000 \ | |
| --eval_steps 50000 \ | |
| --push_to_hub | |