| CMD: -n 4 -g 4 -t 80 -o falcon-7b-ft-mc4_nl_cleaned_tiny/tiny-3e-4lr+1152tbs+1ep+0.1wd -p falcon-7b-ft-mc4_nl_cleaned -e --preprocessed_dataset /dodrio/scratch/projects/2023_005/llm-finetuning/preprocessed_datasets/mc4_nl_cleaned--tiny-falcon-40b-2048 --learning_rate 3e-4 --model_name_or_path tiiuae/falcon-7b --per_device_train_batch_size 12 --per_device_eval_batch_size 24 --gradient_accumulation_steps 6 --eval_accumulation_steps 6 --save_total_limit 3 --eval_steps 170 --save_steps 170 --logging_first_step --weight_decay 0.1 --lr_scheduler_type cosine --early_stopping_patience 5 --warmup_ratio 0.03 --deepspeed ds_config_zero2.json --report_to none | |