lomahony commited on
Commit
cf1d221
1 Parent(s): 6abd1e5

add config.json

Browse files
Files changed (1) hide show
  1. config.json +51 -0
config.json ADDED
@@ -0,0 +1,51 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "seed": 0,
3
+ "exp_name": "pythia_SFT_ga4",
4
+ "batch_size": 64,
5
+ "eval_batch_size": 32,
6
+ "debug": false,
7
+ "fsdp_port": 12355,
8
+ "datasets": [
9
+ "hh"
10
+ ],
11
+ "wandb": {
12
+ "enabled": true,
13
+ "entity": "pythia_dpo",
14
+ "project": "Pythia_LOM"
15
+ },
16
+ "local_dirs": [
17
+ "/scr-ssd",
18
+ "/scr",
19
+ ".cache"
20
+ ],
21
+ "sample_during_eval": false,
22
+ "n_eval_model_samples": 16,
23
+ "do_first_eval": true,
24
+ "local_run_dir": ".cache/laura/pythia_SFT_ga4_2023-07-16_16-50-13_244945",
25
+ "lr": 0.000001,
26
+ "gradient_accumulation_steps": 4,
27
+ "max_grad_norm": 10,
28
+ "max_length": 512,
29
+ "max_prompt_length": 256,
30
+ "n_epochs": 1,
31
+ "n_examples": null,
32
+ "n_eval_examples": 256,
33
+ "trainer": "FSDPTrainer",
34
+ "optimizer": "RMSprop",
35
+ "warmup_steps": 150,
36
+ "activation_checkpointing": false,
37
+ "eval_every": 19968,
38
+ "minimum_log_interval_secs": 1,
39
+ "model": {
40
+ "name_or_path": "EleutherAI/pythia-70m",
41
+ "tokenizer_name_or_path": null,
42
+ "archive": null,
43
+ "block_name": "GPTNeoXLayer",
44
+ "policy_dtype": "float32",
45
+ "fsdp_policy_mp": "bfloat16",
46
+ "reference_dtype": "float16"
47
+ },
48
+ "loss": {
49
+ "name": "sft"
50
+ }
51
+ }