Update app.py

app.py

@@ -54,35 +54,37 @@ model_pipeline = pipeline(
 )
 
 # Use the pipeline in HuggingFacePipeline
-#llm = HuggingFacePipeline(pipeline=model_pipeline)
+llm = HuggingFacePipeline(pipeline=model_pipeline)
 
 ##### Alternative
 from transformers import pipeline
 import torch
 from transformers import AutoTokenizer, AutoModelForCausalLM, BitsAndBytesConfig
 
-READER_MODEL_NAME = "HuggingFaceH4/zephyr-7b-beta"
-
-bnb_config = BitsAndBytesConfig(
-    load_in_4bit=True,
-    bnb_4bit_use_double_quant=True,
-    bnb_4bit_quant_type="nf4",
-    bnb_4bit_compute_dtype=torch.bfloat16,
-)
-rmodel = AutoModelForCausalLM.from_pretrained(READER_MODEL_NAME, quantization_config=bnb_config)
-tokenizer = AutoTokenizer.from_pretrained(READER_MODEL_NAME)
-
-llm = pipeline(
-    model=rmodel,
-    tokenizer=tokenizer,
-    task="text-generation",
-    do_sample=True,
-    temperature=0.2,
-    repetition_penalty=1.1,
-    return_full_text=False,
-    max_new_tokens=500,
-)
+#READER_MODEL_NAME = "HuggingFaceH4/zephyr-7b-beta"
+
+#bnb_config = BitsAndBytesConfig(
+#    load_in_4bit=True,
+#    bnb_4bit_use_double_quant=True,
+#    bnb_4bit_quant_type="nf4",
+#    bnb_4bit_compute_dtype=torch.bfloat16,
+#)
+#rmodel = AutoModelForCausalLM.from_pretrained(READER_MODEL_NAME, quantization_config=bnb_config)
+#tokenizer = AutoTokenizer.from_pretrained(READER_MODEL_NAME)
+
+#llm = pipeline(
+#    model=rmodel,
+#    tokenizer=tokenizer,
+#    task="text-generation",
+#    do_sample=True,
+#    temperature=0.2,
+#    repetition_penalty=1.1,
+#    return_full_text=False,
+#    max_new_tokens=500,
+#)
 #####
+from huggingface_hub import InferenceClient
+
 #repo_id = "meta-llama/Meta-Llama-3-8B-Instruct"
 
 #llm_client = InferenceClient(model=repo_id, timeout=120)