Spaces:
Sleeping
Sleeping
Commit
•
971c779
1
Parent(s):
d56e2db
Update app.py
Browse files
app.py
CHANGED
|
@@ -37,6 +37,10 @@ def load_model(model_id):
|
|
| 37 |
dim=384,
|
| 38 |
depth=8,
|
| 39 |
heads=6,
|
|
|
|
|
|
|
|
|
|
|
|
|
| 40 |
),
|
| 41 |
text_num_embeds=fast_tokenizer_object.vocab_size,
|
| 42 |
tokenizer=tokenizer,
|
|
@@ -49,6 +53,10 @@ def load_model(model_id):
|
|
| 49 |
dim=512,
|
| 50 |
depth=12,
|
| 51 |
heads=6,
|
|
|
|
|
|
|
|
|
|
|
|
|
| 52 |
),
|
| 53 |
text_num_embeds=fast_tokenizer_object.vocab_size,
|
| 54 |
tokenizer=tokenizer,
|
|
|
|
| 37 |
dim=384,
|
| 38 |
depth=8,
|
| 39 |
heads=6,
|
| 40 |
+
attn_kwargs=dict(
|
| 41 |
+
gate_value_heads=True,
|
| 42 |
+
flash = True,
|
| 43 |
+
),
|
| 44 |
),
|
| 45 |
text_num_embeds=fast_tokenizer_object.vocab_size,
|
| 46 |
tokenizer=tokenizer,
|
|
|
|
| 53 |
dim=512,
|
| 54 |
depth=12,
|
| 55 |
heads=6,
|
| 56 |
+
attn_kwargs=dict(
|
| 57 |
+
gate_value_heads=True,
|
| 58 |
+
flash = True,
|
| 59 |
+
),
|
| 60 |
),
|
| 61 |
text_num_embeds=fast_tokenizer_object.vocab_size,
|
| 62 |
tokenizer=tokenizer,
|