Transformers
Inference Endpoints
exnx commited on
Commit
de0c744
1 Parent(s): e1bee50

add weights and config for 1M model

Browse files
Files changed (2) hide show
  1. config.json +25 -0
  2. weights.ckpt +3 -0
config.json ADDED
@@ -0,0 +1,25 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "d_model": 256,
3
+ "n_layer": 8,
4
+ "d_inner": 1024,
5
+ "vocab_size": 12,
6
+ "resid_dropout": 0.0,
7
+ "embed_dropout": 0.1,
8
+ "fused_mlp": false,
9
+ "fused_dropout_add_ln": true,
10
+ "residual_in_fp32": true,
11
+ "pad_vocab_size_multiple": 8,
12
+ "return_hidden_state": true,
13
+ "layer": {
14
+ "_name_": "hyena",
15
+ "emb_dim": 5,
16
+ "filter_order": 64,
17
+ "local_order": 3,
18
+ "l_max": 1000002,
19
+ "modulate": true,
20
+ "w": 10,
21
+ "lr": 6e-4,
22
+ "wd": 0.0,
23
+ "lr_pos_emb": 0.0
24
+ }
25
+ }
weights.ckpt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:5ecc8a121feb9906ff3c11e671b8315156f0e595412ae8e5680781b0a0648104
3
+ size 270921777