wandb: https://wandb.ai/open-assistant/supervised-finetuning/runs/20enq5u1
Config:
oasst_export_latin_cyrillic_alpaca:
datasets:
- oasst_export:
lang: "bg,ca,cs,da,de,en,es,fr,hr,hu,it,nl,pl,pt,ro,ru,sl,sr,sv,uk"
#top_k: 2
input_file_path: 2023-03-25_oasst_research_ready_synth_labels.jsonl.gz
- alpaca
sort_by_length: false
use_custom_sampler: false
pythia-12b:
fp16: true
use_flash_attention: true
residual_dropout: 0.2
learning_rate: 6e-6
model_name: EleutherAI/pythia-12b-deduped
output_dir: pythia_model_12b
weight_decay: 0.0
max_length: 2048
warmup_steps: 100
gradient_checkpointing: false
gradient_accumulation_steps: 4
per_device_train_batch_size: 2
per_device_eval_batch_size: 2
eval_steps: 200
save_steps: 1000
num_train_epochs: 8
save_total_limit: 4
Command used:
deepspeed trainer_sft.py --configs defaults oasst_export_latin_cyrillic_alpaca pythia-12b --cache_dir .cache/ --output_dir .saved_models/oasst-sft-2_12b --deepspeed
- Downloads last month
- 12