Update README.md

README.md

@@ -7,10 +7,6 @@ tags:
 - merge
 
 ---
-# merge
-
-This is a merge of pre-trained language models created using [mergekit](https://github.com/cg123/mergekit).
-
 ## Merge Details
 ### Merge Method
 
@@ -65,4 +61,26 @@ from transformers import AutoTokenizer, AutoModelForCausalLM
 tokenizer = AutoTokenizer.from_pretrained("ayoubkirouane/Mistral-Merged7B")
 model = AutoModelForCausalLM.from_pretrained("ayoubkirouane/Mistral-Merged7B")
 
+# 4 bit : 
+
+from transformers import AutoModelForCausalLM, AutoTokenizer, BitsAndBytesConfig
+import torch
+
+nf4_config = BitsAndBytesConfig(
+   load_in_4bit=True,
+   bnb_4bit_quant_type="nf4",
+   bnb_4bit_use_double_quant=True,
+   bnb_4bit_compute_dtype=torch.bfloat16
+)
+model = AutoModelForCausalLM.from_pretrained(
+    "ayoubkirouane/Mistral-SLERP-Merged7B",
+    device_map='auto',
+    quantization_config=nf4_config,
+    use_cache=False
+)
+tokenizer = AutoTokenizer.from_pretrained("ayoubkirouane/Mistral-SLERP-Merged7B")
+
+tokenizer.pad_token = tokenizer.eos_token
+tokenizer.padding_side = "right"
+
 ```