bourdoiscatie commited on
Commit
7e059b8
1 Parent(s): d4f4d94

Update config.json

Browse files
Files changed (1) hide show
  1. config.json +18 -11
config.json CHANGED
@@ -4,13 +4,18 @@
4
  "FlashT5ForConditionalGeneration"
5
  ],
6
  "attention_dropout_rate": 0.0,
 
 
7
  "auto_map": {
8
  "AutoConfig": "configuration_flash_t5.FlashT5Config",
9
  "AutoModel": "modeling_flash_t5.FlashT5ForConditionalGeneration",
10
- "AutoModelForSeq2SeqLM": "modeling_flash_t5.FlashT5ForConditionalGeneration"
 
 
 
11
  },
12
  "classifier_dropout": 0.0,
13
- "d_ff": 2048,
14
  "d_kv": 64,
15
  "d_model": 512,
16
  "decoder_start_token_id": 0,
@@ -18,16 +23,17 @@
18
  "dropout_rate": 0.0,
19
  "eos_token_id": 1,
20
  "feed_forward_proj": "relu",
 
21
  "initializer_factor": 1.0,
22
  "is_encoder_decoder": false,
23
  "is_gated_act": false,
24
- "label_smoothing": 0.0,
25
  "layer_norm_epsilon": 1e-06,
26
  "max_sequence_length": 1024,
27
  "model_type": "flash_t5",
28
- "num_decoder_layers": 12,
29
- "num_heads": 8,
30
- "num_layers": 12,
31
  "pad_token_id": 0,
32
  "position_encoding_type": "t5",
33
  "relative_attention_max_distance": 128,
@@ -38,16 +44,17 @@
38
  "rotary_scale_base": null,
39
  "tie_word_embeddings": false,
40
  "torch_dtype": "float32",
41
- "transformers_version": "4.37.2",
42
  "use_cache": true,
43
- "use_flash_attention": "ref",
44
  "use_full_bias_size": false,
45
  "use_gelu_act": true,
46
  "use_glu_mlp": true,
 
47
  "use_randomized_position_encoding": false,
48
- "use_triton_crossentropy": true,
49
  "use_triton_gated_mlp": false,
50
  "use_triton_layernorm": false,
51
- "vocab_size": 32768,
52
  "z_loss": 0.0001
53
- }
 
4
  "FlashT5ForConditionalGeneration"
5
  ],
6
  "attention_dropout_rate": 0.0,
7
+ "attention_scale": 1.0,
8
+ "attention_type": "ref",
9
  "auto_map": {
10
  "AutoConfig": "configuration_flash_t5.FlashT5Config",
11
  "AutoModel": "modeling_flash_t5.FlashT5ForConditionalGeneration",
12
+ "AutoModelForQuestionAnswering": "custom_heads_flash_t5.FlashT5ForQuestionAnswering",
13
+ "AutoModelForSeq2SeqLM": "modeling_flash_t5.FlashT5ForConditionalGeneration",
14
+ "AutoModelForSequenceClassification": "custom_heads_flash_t5.FlashT5ForSequenceClassification",
15
+ "AutoModelForTokenClassification": "custom_heads_flash_t5.FlashT5ForTokenClassification"
16
  },
17
  "classifier_dropout": 0.0,
18
+ "d_ff": 1024,
19
  "d_kv": 64,
20
  "d_model": 512,
21
  "decoder_start_token_id": 0,
 
23
  "dropout_rate": 0.0,
24
  "eos_token_id": 1,
25
  "feed_forward_proj": "relu",
26
+ "fire_mlp_width": 32,
27
  "initializer_factor": 1.0,
28
  "is_encoder_decoder": false,
29
  "is_gated_act": false,
30
+ "label_smoothing": 0.1,
31
  "layer_norm_epsilon": 1e-06,
32
  "max_sequence_length": 1024,
33
  "model_type": "flash_t5",
34
+ "num_decoder_layers": 8,
35
+ "num_heads": 6,
36
+ "num_layers": 8,
37
  "pad_token_id": 0,
38
  "position_encoding_type": "t5",
39
  "relative_attention_max_distance": 128,
 
44
  "rotary_scale_base": null,
45
  "tie_word_embeddings": false,
46
  "torch_dtype": "float32",
47
+ "transformers_version": "4.39.3",
48
  "use_cache": true,
49
+ "use_flash_attention": "triton",
50
  "use_full_bias_size": false,
51
  "use_gelu_act": true,
52
  "use_glu_mlp": true,
53
+ "use_masking": false,
54
  "use_randomized_position_encoding": false,
55
+ "use_triton_crossentropy": false,
56
  "use_triton_gated_mlp": false,
57
  "use_triton_layernorm": false,
58
+ "vocab_size": 32128,
59
  "z_loss": 0.0001
60
+ }