call t5 model class before assigning to LLM pipeline

app.py

@@ -13,6 +13,7 @@ import json
 import torch
 from tqdm.auto import tqdm
 from transformers import BitsAndBytesConfig
+from transformers import AutoModelForCausalLM, AutoTokenizer, pipeline, T5Tokenizer, AutoModel, T5ForConditionalGeneration
 from langchain.text_splitter import RecursiveCharacterTextSplitter
 
 
@@ -97,14 +98,27 @@ def load_faiss_index():
 
 @st.cache_resource
 def load_llm_model():
-    #this one is for running with GPT
-    llm = HuggingFacePipeline.from_model_id(model_id= 'lmsys/fastchat-t5-3b-v1.0', 
-                                            task= 'text2text-generation',
-                                            model_kwargs={ 
-                                                # "device_map": "auto",
-                                                        "max_length": 256, "temperature": 0,
-                                                        "repetition_penalty": 1.5,
-                                                         "quantization_config": bitsandbyte_config}) #add this quantization config
+    #this one is for running with GPU
+
+    model = T5ForConditionalGeneration.from_pretrained(model_id='lmsys/fastchat-t5-3b-v1.0',
+                                                       quantization_config = bitsandbyte_config,
+                                                       device_map= 'auto')
+    tokenizer =  AutoTokenizer.from_pretrained(core_model_name)
+
+
+    pipe = pipeline(
+        task= 'text2text-generation', model=model, tokenizer=tokenizer, max_new_tokens=256, model_kwargs={"temperature":0,
+                                                                                                         "repetition_penalty": 1.5}
+    )
+    llm = HuggingFacePipeline(pipeline=pipe)
+    
+    # llm = HuggingFacePipeline.from_model_id(model_id= 'lmsys/fastchat-t5-3b-v1.0', 
+    #                                         task= 'text2text-generation',
+    #                                         model_kwargs={ 
+    #                                             # "device_map": "auto",
+    #                                                     "max_length": 256, "temperature": 0,
+    #                                                     "repetition_penalty": 1.5,
+    #                                                      "quantization_config": bitsandbyte_config}) #add this quantization config
     
     
     # llm = HuggingFacePipeline.from_model_id(model_id= 'lmsys/fastchat-t5-3b-v1.0',