dataset:
  name: alpaca_clean
  dataset_config:
    name: alpaca
    path: yahma/alpaca-cleaned
    chunk_size: 1024  # sequence length for distilling
    concat_data: true
    cache_dir: 'data/alpaca'  # Change this to where you want to save
  pretrained_model_config:
    pretrained_model_name_or_path: 'mistralai/Mistral-7B-v0.1'  # will be updated based on model_config
    cache_dir: '/scr-ssd/mzhang/models/mistral-v0.1'
  preprocess_config: null

dataloader:
  batch_size: 1
  num_workers: 2
  drop_last: false
  pin_memory: true

optimizer:
  optim: adamw_torch_fused
  lr: 0.01
  weight_decay: 0.0

lr_scheduler:
  lr_scheduler_type: none

trainer:  # HuggingFace Trainer-like arguments  
  name: null