lomahony commited on
Commit
3f2d1c9
1 Parent(s): 98e817d

First model version

Browse files
Files changed (4) hide show
  1. config.yaml +44 -0
  2. optimizer.pt +3 -0
  3. policy.pt +3 -0
  4. scheduler.pt +3 -0
config.yaml ADDED
@@ -0,0 +1,44 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ seed: 0
2
+ exp_name: pythia410m_hh_ga4_dpo
3
+ batch_size: 64
4
+ eval_batch_size: 32
5
+ debug: false
6
+ fsdp_port: 12355
7
+ datasets:
8
+ - hh
9
+ wandb:
10
+ enabled: true
11
+ entity: pythia_dpo
12
+ project: Pythia_LOM
13
+ local_dirs:
14
+ - /scr-ssd
15
+ - /scr
16
+ - .cache
17
+ sample_during_eval: false
18
+ n_eval_model_samples: 16
19
+ do_first_eval: true
20
+ local_run_dir: .cache/laura/pythia410m_hh_ga4_dpo_2023-07-19_13-02-35_532776
21
+ lr: 1.0e-06
22
+ gradient_accumulation_steps: 4
23
+ max_grad_norm: 10.0
24
+ max_length: 512
25
+ max_prompt_length: 256
26
+ n_epochs: 1
27
+ n_examples: null
28
+ n_eval_examples: 256
29
+ trainer: FSDPTrainer
30
+ optimizer: RMSprop
31
+ warmup_steps: 150
32
+ activation_checkpointing: false
33
+ eval_every: 19968
34
+ minimum_log_interval_secs: 1.0
35
+ model:
36
+ name_or_path: EleutherAI/pythia-410m
37
+ tokenizer_name_or_path: null
38
+ archive: null
39
+ block_name: GPTNeoXLayer
40
+ policy_dtype: float32
41
+ fsdp_policy_mp: bfloat16
42
+ reference_dtype: float16
43
+ loss:
44
+ name: sft
optimizer.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:d61e9f7ae162b6ca0f3e6f4310f307a45d8d5f0718a757228d7f9e8d877108df
3
+ size 1621435717
policy.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:f7d5342d3950db163bc8bacb6d0a7eac25502526a3066e30cfd134a6f9da6d9a
3
+ size 1722144187
scheduler.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:ef3bb892ec5f85ebcd911ec0358781fc843702c92bef54eec6bc38b127a23878
3
+ size 627