Create README.md

README.md

@@ -0,0 +1,29 @@
+# Mistral-7B-Instruct-v0.1-8bit
+
+Create model
+```python
+model_path = "mistralai/Mistral-7B-Instruct-v0.1"
+bnb_config = BitsAndBytesConfig(
+    load_in_8bit=True
+)
+model = AutoModelForCausalLM.from_pretrained(model_path, trust_remote_code=True, quantization_config=bnb_config, device_map="auto")
+tokenizer = AutoTokenizer.from_pretrained(model_path)
+```
+
+Load in pipeline
+```python
+text_generation_pipeline = transformers.pipeline(
+    model=model,
+    tokenizer=tokenizer,
+    task="text-generation",
+    eos_token_id=tokenizer.eos_token_id,
+    pad_token_id=tokenizer.eos_token_id,
+    repetition_penalty=1.1,
+    return_full_text=True,
+    max_new_tokens=100,
+)
+mistral_llm = HuggingFacePipeline(pipeline=text_generation_pipeline)
+
+text = "what is mistral?"
+mistral_llm.invoke(text)
+```