KevinG commited on
Commit
4b647a7
1 Parent(s): 655bb82

Upload ScriptableLMForPreTraining

Browse files
Files changed (2) hide show
  1. config.json +70 -0
  2. pytorch_model.bin +3 -0
config.json ADDED
@@ -0,0 +1,70 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "arch": {
3
+ "architectures": [
4
+ "ScriptableMaskedLM"
5
+ ],
6
+ "attention": {
7
+ "causal_attention": false,
8
+ "dropout_prob": 0.1,
9
+ "high_level_fusion": false,
10
+ "low_level_fusion": true,
11
+ "num_attention_heads": 12,
12
+ "qkv_bias": false,
13
+ "rotary_embedding": false,
14
+ "seq_op_in_fp32": false,
15
+ "sequence_op": "torch-softmax",
16
+ "skip_output_projection": false,
17
+ "type": "flash-attention-impl"
18
+ },
19
+ "classification_head": {
20
+ "classifier_dropout": 0.1,
21
+ "head_dim": 1024,
22
+ "include_ff_layer": true,
23
+ "nonlin": "Tanh",
24
+ "pooler": "zero_index"
25
+ },
26
+ "decoder_bias": false,
27
+ "deepnorm_scaling": false,
28
+ "embedding": {
29
+ "dropout_prob": 0.1,
30
+ "embedding_dim": 768,
31
+ "max_seq_length": 128,
32
+ "normalization": true,
33
+ "pad_token_id": 0,
34
+ "pos_embedding": "learned",
35
+ "vocab_size": 32768
36
+ },
37
+ "ffn_layer_frequency": 1,
38
+ "final_norm": true,
39
+ "gradient_checkpointing": false,
40
+ "hidden_dropout_prob": 0.1,
41
+ "hidden_size": 768,
42
+ "init": {
43
+ "std": 0.02,
44
+ "type": "normal"
45
+ },
46
+ "intermed_size": 3072,
47
+ "layer_drop_theta": null,
48
+ "layer_fusion": true,
49
+ "layer_macro_type": "transformer",
50
+ "loss": "cross-entropy",
51
+ "nonlin": "GELUglu",
52
+ "norm": "LayerNorm",
53
+ "norm_eps": 1e-12,
54
+ "norm_scheme": "pre",
55
+ "num_labels": null,
56
+ "num_transformer_layers": 16,
57
+ "recurrent_layers": null,
58
+ "skip_head_transform": true,
59
+ "sparse_prediction": true,
60
+ "tie_weights": true,
61
+ "use_bias": false,
62
+ "z_loss_factor": 0
63
+ },
64
+ "architectures": [
65
+ "ScriptableLMForPreTraining"
66
+ ],
67
+ "model_type": "crammedBERT",
68
+ "torch_dtype": "float32",
69
+ "transformers_version": "4.26.1"
70
+ }
pytorch_model.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:78f2a16f83aaf52b9522cf59f515fd6ddb9d9cd9eb9a94fb48730e59a7612967
3
+ size 481161743