Update README.md
Browse files
README.md
CHANGED
@@ -37,24 +37,4 @@ experts:
|
|
37 |
|
38 |
## 💻 Usage
|
39 |
|
40 |
-
|
41 |
-
!pip install -qU transformers bitsandbytes accelerate
|
42 |
-
|
43 |
-
from transformers import AutoTokenizer
|
44 |
-
import transformers
|
45 |
-
import torch
|
46 |
-
|
47 |
-
model = "mlabonne/phixtral-4x2.8"
|
48 |
-
|
49 |
-
tokenizer = AutoTokenizer.from_pretrained(model)
|
50 |
-
pipeline = transformers.pipeline(
|
51 |
-
"text-generation",
|
52 |
-
model=model,
|
53 |
-
model_kwargs={"torch_dtype": torch.float16, "load_in_4bit": True},
|
54 |
-
)
|
55 |
-
|
56 |
-
messages = [{"role": "user", "content": "Explain what a Mixture of Experts is in less than 100 words."}]
|
57 |
-
prompt = pipeline.tokenizer.apply_chat_template(messages, tokenize=False, add_generation_prompt=True)
|
58 |
-
outputs = pipeline(prompt, max_new_tokens=256, do_sample=True, temperature=0.7, top_k=50, top_p=0.95)
|
59 |
-
print(outputs[0]["generated_text"])
|
60 |
-
```
|
|
|
37 |
|
38 |
## 💻 Usage
|
39 |
|
40 |
+
This architecture is not compatible with the transformers library. I'm working on hacking something to run it. Contact me if you're interested!
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|