{
  "seed": 0,
  "exp_name": "pythia_SFT_ga4",
  "batch_size": 64,
  "eval_batch_size": 32,
  "debug": false,
  "fsdp_port": 12355,
  "datasets": [
    "hh"
  ],
  "wandb": {
    "enabled": true,
    "entity": "pythia_dpo",
    "project": "Pythia_LOM"
  },
  "local_dirs": [
    "/scr-ssd",
    "/scr",
    ".cache"
  ],
  "sample_during_eval": false,
  "n_eval_model_samples": 16,
  "do_first_eval": true,
  "local_run_dir": ".cache/laura/pythia_SFT_ga4_2023-07-16_16-50-13_244945",
  "lr": 0.000001,
  "gradient_accumulation_steps": 4,
  "max_grad_norm": 10,
  "max_length": 512,
  "max_prompt_length": 256,
  "n_epochs": 1,
  "n_examples": null,
  "n_eval_examples": 256,
  "trainer": "FSDPTrainer",
  "optimizer": "RMSprop",
  "warmup_steps": 150,
  "activation_checkpointing": false,
  "eval_every": 19968,
  "minimum_log_interval_secs": 1,
  "model": {
    "name_or_path": "EleutherAI/pythia-70m",
    "tokenizer_name_or_path": null,
    "archive": null,
    "block_name": "GPTNeoXLayer",
    "policy_dtype": "float32",
    "fsdp_policy_mp": "bfloat16",
    "reference_dtype": "float16"
  },
  "loss": {
    "name": "sft"
  }
}