Update app.py

app.py

@@ -1,6 +1,7 @@
 import gradio as gr
 import torch
 from transformers import pipeline
+from ctransformers import AutoModelForCausalLM
 
 MODEL_NAME = "openai/whisper-tiny"
 BATCH_SIZE = 8
@@ -21,6 +22,11 @@ def transcribe(inputs, task = "transcribe"):
     text = pipe(inputs, batch_size=BATCH_SIZE, generate_kwargs={"task": task}, return_timestamps=True)["text"]
     return  text
 
+# Set gpu_layers to the number of layers to offload to GPU. Set to 0 if no GPU acceleration is available on your system.
+llm = AutoModelForCausalLM.from_pretrained("TheBloke/Mistral-7B-v0.1-GGUF", model_file="mistral-7b-v0.1.Q4_K_M.gguf", model_type="mistral", gpu_layers=0)
+
+print(llm("AI is going to"))
+
 iface = gr.Interface(
     fn=transcribe,
     inputs=[