andreaskoepf/oasst-sft-2-pythia-12b-4000

wandb: https://wandb.ai/open-assistant/supervised-finetuning/runs/20enq5u1

Config:

oasst_export_latin_cyrillic_alpaca:
  datasets:
    - oasst_export:
        lang: "bg,ca,cs,da,de,en,es,fr,hr,hu,it,nl,pl,pt,ro,ru,sl,sr,sv,uk"
        #top_k: 2
        input_file_path: 2023-03-25_oasst_research_ready_synth_labels.jsonl.gz
    - alpaca 
  sort_by_length: false
  use_custom_sampler: false

pythia-12b:
  fp16: true
  use_flash_attention: true
  residual_dropout: 0.2
  learning_rate: 6e-6
  model_name: EleutherAI/pythia-12b-deduped
  output_dir: pythia_model_12b
  weight_decay: 0.0
  max_length: 2048
  warmup_steps: 100
  gradient_checkpointing: false
  gradient_accumulation_steps: 4
  per_device_train_batch_size: 2
  per_device_eval_batch_size: 2
  eval_steps: 200
  save_steps: 1000
  num_train_epochs: 8
  save_total_limit: 4

Command used: deepspeed trainer_sft.py --configs defaults oasst_export_latin_cyrillic_alpaca pythia-12b --cache_dir .cache/ --output_dir .saved_models/oasst-sft-2_12b --deepspeed