Update README.md

README.md

@@ -24,12 +24,12 @@ language:
 - en
 metrics:
 - accuracy
-- code_eval
-- bleu
 - brier_score
+- code_eval
+pipeline_tag: text-generation
 ---
 
-# LeroyDyer/Mixtral_AI_128K_B_7b
+# LeroyDyer/Mixtral_AI_128K_B
 
 This is a merge of pre-trained language models created using [mergekit](https://github.com/cg123/mergekit).
 
@@ -113,39 +113,52 @@ print(response.text)
 ```
 
 
+```
+pip install transformers==4.34.0
+pip install flash-attn==2.3.1.post1 --no-build-isolation
+pip install accelerate==0.23.0
 
 
-
-# 1. Method1 
-
 ```
 
 
-from transformers import AutoTokenizer, AutoModelForCausalLM
 
-tokenizer = AutoTokenizer.from_pretrained("LeroyDyer/Mixtral_AI_128K_B_7b", trust_remote_code=True)
-model = AutoModelForCausalLM.from_pretrained("LeroyDyer/Mixtral_AI_128K_B_7b", trust_remote_code=True)
+## METHOD 2
 
 
 ```
 
 
-# 2. Method2
-
-```
+from transformers import AutoModelForCausalLM, AutoTokenizer
+import transformers
+import torch
 
+model_id = "LeroyDyer/Mixtral_AI_128K_B"
 
-from transformers import AutoTokenizer, AutoModelForCausalLM
+tokenizer = AutoTokenizer.from_pretrained(model_id)
+model = AutoModelForCausalLM.from_pretrained(model_id,
+                                             torch_dtype=torch.bfloat16,
+                                             use_flash_attention_2=True,
+                                             device_map="auto", trust_remote_code=True)
+pipeline = transformers.pipeline(
+    "text-generation",
+    model=model,
+    tokenizer=tokenizer,
+)
+prompt = "<|prompter|>What are the main challenges to support a long context for LLM?</s><|assistant|>"
+
+sequences = pipeline(
+    prompt,
+    max_new_tokens=400,
+    do_sample=False,
+    return_full_text=False,
+    num_return_sequences=1,
+    eos_token_id=tokenizer.eos_token_id,
+)
+for seq in sequences:
+    print(f"{seq['generated_text']}")
 
-tokenizer = AutoTokenizer.from_pretrained("LeroyDyer/Mixtral_AI_128k_7b-GGUF",
-  use_flash_attention_2=True,
-  torch_dtype=torch.bfloat16,
-device_map="auto", trust_remote_code=True)
 
-model = AutoModelForCausalLM.from_pretrained("LeroyDyer/Mixtral_AI_128k_7b-GGUF",
-  use_flash_attention_2=True,
-  torch_dtype=torch.bfloat16,
-  device_map="auto", trust_remote_code=True)