Update app.py

app.py

@@ -2,8 +2,6 @@ import gradio as gr
 import librosa
 import torch
 from transformers import AutoTokenizer, pipeline, logging
-from auto_gptq import AutoGPTQForCausalLM, BaseQuantizeConfig
-
 from transformers import SpeechT5Processor, SpeechT5ForSpeechToText
 
 
@@ -52,47 +50,24 @@ def audio_to_text(audio, mic_audio=None):
 
 # Text Generation
 
-model_name_or_path = "TheBloke/Llama-2-7b-Chat-GPTQ"
-model_basename = "gptq_model-4bit-128g"
-
-use_triton = False
-
-llama_tokenizer = AutoTokenizer.from_pretrained(model_name_or_path, use_fast=True)
-
-llama_model = AutoGPTQForCausalLM.from_quantized(model_name_or_path,
-        model_basename=model_basename,
-        use_safetensors=True,
-        trust_remote_code=True,
-        device="cuda:0",
-        use_triton=use_triton,
-        quantize_config=None)
+model_path= hf_hub_download(repo_id="TheBloke/Llama-2-7B-Chat-GGML", filename="llama-2-7b-chat.ggmlv3.q4_0.bin")
+llm2 = Llama(model_path=model_path)
 
 def generate(text):
-  prompt = text
   system_message = "You are a helpful, respectful and honest assistant. Always answer as helpfully as possible, while being safe.  Your answers should not include any harmful, unethical, racist, sexist, toxic, dangerous, or illegal content. Please ensure that your responses are socially unbiased and positive in nature. If a question does not make any sense, or is not factually coherent, explain why instead of answering something not correct. If you don't know the answer to a question, please don't share false information. Give short, simple and direct answers"
   prompt_template=f'''[INST] <<SYS>>
   {system_message}
   <</SYS>>
 
-  {prompt} [/INST]'''
-
-  pipe = pipeline(
-      "text-generation",
-      model=model,
-      tokenizer=tokenizer,
-      max_new_tokens=512,
-      temperature=0.7,
-      top_p=0.95,
-      repetition_penalty=1.15
-  )
-
-  return pipe(prompt_template)[0]['generated_text']
+  {text} [/INST]'''
+  chat_compl = llm2.create_completion(prompt=prompt_template, top_k=50, top_p=0.7, temperature=0.7, repeat_penalty=1.5)
+  return chat_compl['choices'][0]['text'].strip()
 
 def audio_text_generate(audio):
   audio_text = audio_to_text(audio)
   generated_text = generate(audio_text)
-  response = generated_text[generated_text.index("[/INST]")+7:].strip()
-  return audio_text, response
+  # response = generated_text[generated_text.index("[/INST]")+7:].strip()
+  return audio_text, generated_text
 
 
 demo = gr.Interface(fn=audio_text_generate,