mobinln commited on
Commit
c2aa89c
1 Parent(s): 6da1c26

fix: change to qwen2

Browse files
Files changed (1) hide show
  1. app.py +11 -2
app.py CHANGED
@@ -1,8 +1,17 @@
1
  import gradio as gr
2
  from llama_cpp import Llama
3
 
4
- model = "Qwen/Qwen1.5-0.5B-Chat-GGUF"
5
- llm = Llama.from_pretrained(repo_id=model, filename="*q8_0.gguf", verbose=True)
 
 
 
 
 
 
 
 
 
6
 
7
 
8
  def respond(
 
1
  import gradio as gr
2
  from llama_cpp import Llama
3
 
4
+ model = "Qwen/Qwen2-0.5B-Instruct-GGUF"
5
+ llm = Llama.from_pretrained(
6
+ repo_id=model,
7
+ filename="*q8_0.gguf",
8
+ verbose=True,
9
+ use_mmap=False,
10
+ use_mlock=True,
11
+ n_threads=2,
12
+ n_threads_batch=2,
13
+ n_ctx=32000,
14
+ )
15
 
16
 
17
  def respond(