NickyNicky
commited on
Commit
•
b623007
1
Parent(s):
025f592
Update README.md
Browse files
README.md
CHANGED
@@ -46,6 +46,18 @@ reference-data-model:
|
|
46 |
```
|
47 |
|
48 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
49 |
## Version
|
50 |
```py
|
51 |
import torch, transformers,torchvision
|
@@ -67,6 +79,9 @@ from transformers import (
|
|
67 |
GenerationConfig,
|
68 |
TextIteratorStreamer,
|
69 |
)
|
|
|
|
|
|
|
70 |
import torch
|
71 |
|
72 |
# model_id = 'Open-Orca/Mistral-7B-OpenOrca'
|
@@ -78,6 +93,9 @@ model = AutoModelForCausalLM.from_pretrained(model_id,
|
|
78 |
torch_dtype=torch.bfloat16,
|
79 |
load_in_4bit=True,
|
80 |
low_cpu_mem_usage= True,
|
|
|
|
|
|
|
81 |
)
|
82 |
|
83 |
max_length=2048
|
|
|
46 |
```
|
47 |
|
48 |
|
49 |
+
##
|
50 |
+
|
51 |
+
|
52 |
+
```py
|
53 |
+
|
54 |
+
#Train activation
|
55 |
+
#https://github.com/tomaarsen/attention_sinks
|
56 |
+
|
57 |
+
pip install attention_sinks
|
58 |
+
```
|
59 |
+
|
60 |
+
|
61 |
## Version
|
62 |
```py
|
63 |
import torch, transformers,torchvision
|
|
|
79 |
GenerationConfig,
|
80 |
TextIteratorStreamer,
|
81 |
)
|
82 |
+
|
83 |
+
from attention_sinks import AutoModelForCausalLM
|
84 |
+
|
85 |
import torch
|
86 |
|
87 |
# model_id = 'Open-Orca/Mistral-7B-OpenOrca'
|
|
|
93 |
torch_dtype=torch.bfloat16,
|
94 |
load_in_4bit=True,
|
95 |
low_cpu_mem_usage= True,
|
96 |
+
|
97 |
+
attention_sink_size=4,
|
98 |
+
attention_sink_window_size=1024, #512, # <- Low for the sake of faster generation
|
99 |
)
|
100 |
|
101 |
max_length=2048
|