jefsnacker commited on
Commit
4e6364b
1 Parent(s): d6dbafc

upload new gpt weights

Browse files
Files changed (2) hide show
  1. micro_gpt_config.yaml +47 -0
  2. micro_gpt_weights.pt +3 -0
micro_gpt_config.yaml ADDED
@@ -0,0 +1,47 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ attn_pdrop: 0.1
2
+ b1: 0.9
3
+ b2: 0.95
4
+ batch_size: 1024
5
+ blocks: 6
6
+ d_model: 128
7
+ embd_pdrop: 0.1
8
+ epochs: 500000
9
+ heads: 4
10
+ lr: 0.0003
11
+ model_type: gpt
12
+ num_workers: 4
13
+ resid_pdrop: 0.1
14
+ stoi:
15
+ ' ': 1
16
+ '''': 2
17
+ '-': 3
18
+ .: 0
19
+ a: 4
20
+ b: 5
21
+ c: 6
22
+ d: 7
23
+ e: 8
24
+ f: 9
25
+ g: 10
26
+ h: 11
27
+ i: 12
28
+ j: 13
29
+ k: 14
30
+ l: 15
31
+ m: 16
32
+ n: 17
33
+ o: 18
34
+ p: 19
35
+ q: 20
36
+ r: 21
37
+ s: 22
38
+ t: 23
39
+ u: 24
40
+ v: 25
41
+ w: 26
42
+ x: 27
43
+ y: 28
44
+ z: 29
45
+ vocab: 30
46
+ weight_decay: 0.1
47
+ window: 32
micro_gpt_weights.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:6584b4706a04079483a29e40028bb22945471961b01194d584806575ea8cd8df
3
+ size 4859909