Dan Fu commited on
Commit
7f3a69e
1 Parent(s): 624c2de

Initial model

Browse files
Files changed (4) hide show
  1. config.json +4 -0
  2. config.yaml +38 -0
  3. model.pt +3 -0
  4. version.txt +1 -0
config.json ADDED
@@ -0,0 +1,4 @@
 
 
 
 
 
1
+ {
2
+ "model_type": "m2_bert"
3
+ }
4
+
config.yaml ADDED
@@ -0,0 +1,38 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Note that some of the fields in this template haven't been filled in yet.
2
+ # Please resolve any `null` fields before launching!
3
+
4
+ precision: amp_bf16
5
+ max_seq_len: 8192
6
+
7
+ # Tokenizer for dataset creation
8
+ tokenizer_name: bert-base-uncased
9
+
10
+ # Base model config
11
+ model:
12
+ name: bert
13
+ pretrained_model_name: ${tokenizer_name}
14
+ tokenizer_name: ${tokenizer_name}
15
+ model_config:
16
+ num_attention_heads: 12
17
+ num_hidden_layers: 12
18
+ attention_probs_dropout_prob: 0.0
19
+ max_position_embeddings: 8192
20
+
21
+ monarch_mixer_sequence_mixing: True
22
+ long_conv_l_max: 8192
23
+ long_conv_kernel_learning_rate: 1e-3
24
+ hyena_lr_pos_emb: 1e-5
25
+ hyena_w: 10
26
+ hyena_wd: 0.1
27
+ hyena_emb_dim: 5
28
+ hyena_filter_order: 128
29
+ hyena_training_additions: False
30
+
31
+ bidirectional: true
32
+ residual_long_conv: true
33
+
34
+ use_glu_mlp: True
35
+ use_monarch_mlp: True
36
+ monarch_mlp_nblocks: 4
37
+ use_positional_encodings: True
38
+
model.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:e7155c21c85cc6a169064a007fad01d2a38b8e1e3d7c98b300f2b2dfaace01b4
3
+ size 1050873856
version.txt ADDED
@@ -0,0 +1 @@
 
 
1
+ 1