Update app.py

app.py

@@ -1,4 +1,4 @@
-from ctransformers import AutoModelForCausalLM
+#from ctransformers import AutoModelForCausalLM
 import re, requests, json
 import gradio as gr
 import random
@@ -23,14 +23,21 @@ MAX_NEW_TOKENS = 25
 GPU_LAYERS = 0
 STOP_LIST=["###","##"]
 
-stopping_criteria = StoppingCriteriaList([MaxLengthCriteria(max_length=64)])
+#stopping_criteria = StoppingCriteriaList([MaxLengthCriteria(max_length=64)])
 
+from huggingface_hub import hf_hub_download
+hf_hub_download(repo_id="gorkemgoknar/llama2-7f-moviechatbot-ggml-q4", local_dir=".", filename="llama2-7f-fp16-ggml-q4.bin")
+model_path="./llama2-7f-fp16-ggml-q4.bin"
 
-llm = AutoModelForCausalLM.from_pretrained("gorkemgoknar/llama2-7f-moviechatbot-ggml-q4", 
-                                           model_type='llama',
-                                           gpu_layers=GPU_LAYERS,
-                                           max_new_tokens=MAX_NEW_TOKENS, 
-                                           stop=STOP_LIST)
+llm = Llama(model_path=model_path,n_gpu_layers=0)
+
+
+# to use with ctransfomers
+#llm = AutoModelForCausalLM.from_pretrained("gorkemgoknar/llama2-7f-moviechatbot-ggml-q4", 
+#                                           model_type='llama',
+#                                           gpu_layers=GPU_LAYERS,
+#                                           max_new_tokens=MAX_NEW_TOKENS, 
+#                                           stop=STOP_LIST)
 
 
 
@@ -79,7 +86,9 @@ def get_audio_url(text,character):
     
 def get_response_cpp(prompt):
 
-    response_text= llm(prompt)
+    #response_text= llm(prompt)
+    output = llm("### Context: talks friendly### History: ### Morpheus: I challenge you to battle of words!### Gerald:", max_tokens=32, stop=["#","sierpeda"], echo=True)
+    response_Text= output["choices"][0]["text"]
 
     return response_text