Update main.py

main.py

@@ -4,10 +4,10 @@ from transformers import AutoModelForCausalLM, AutoTokenizer, pipeline,BitsAndBy
 # import bitsandbytes
 from langchain_core.prompts import PromptTemplate
 
-# quants = BitsAndBytesConfig(load_in_4bit=True)
+quants = BitsAndBytesConfig(load_in_4bit=True)
 model_id = "mistralai/Mistral-7B-Instruct-v0.2"
-tokenizer = AutoTokenizer.from_pretrained(model_id,load_in_8bit=True)
-model = AutoModelForCausalLM.from_pretrained(model_id,load_in_8bit=True)
+tokenizer = AutoTokenizer.from_pretrained(model_id,qunatization_config = quants)
+model = AutoModelForCausalLM.from_pretrained(model_id,qunatization_config = quants)
 pipe = pipeline("text-generation", model=model, tokenizer=tokenizer)
 hf = HuggingFacePipeline(pipeline=pipe)