aexyb commited on
Commit
9a2b31d
1 Parent(s): 1d2c609

Update app.py

Browse files
Files changed (1) hide show
  1. app.py +62 -16
app.py CHANGED
@@ -1,17 +1,63 @@
1
  import gradio as gr
2
- import concurrent.futures
3
-
4
- # Load the model into RAM
5
- model = gr.load("models/TheBloke/SOLAR-10.7B-Instruct-v1.0-uncensored-GGUF")
6
-
7
- def interact(input):
8
- # Define the function for user interaction
9
- response = model(input)
10
- return response
11
-
12
- # Use ThreadPoolExecutor to manage the threads
13
- with concurrent.futures.ThreadPoolExecutor(max_workers=2) as executor:
14
- # Create a Gradio interface with the loaded model
15
- interface = gr.Interface(fn=interact, inputs="text", outputs="text")
16
- # Handle the interactions with Gradio
17
- interface.launch()
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
  import gradio as gr
2
+ from huggingface_hub import InferenceClient
3
+
4
+ """
5
+ For more information on `huggingface_hub` Inference API support, please check the docs: https://huggingface.co/docs/huggingface_hub/v0.22.2/en/guides/inference
6
+ """
7
+ client = InferenceClient("TheBloke/SOLAR-10.7B-Instruct-v1.0-uncensored-GGUF")
8
+
9
+
10
+ def respond(
11
+ message,
12
+ history: list[tuple[str, str]],
13
+ system_message,
14
+ max_tokens,
15
+ temperature,
16
+ top_p,
17
+ ):
18
+ messages = [{"role": "system", "content": system_message}]
19
+
20
+ for val in history:
21
+ if val[0]:
22
+ messages.append({"role": "user", "content": val[0]})
23
+ if val[1]:
24
+ messages.append({"role": "assistant", "content": val[1]})
25
+
26
+ messages.append({"role": "user", "content": message})
27
+
28
+ response = ""
29
+
30
+ for message in client.chat_completion(
31
+ messages,
32
+ max_tokens=max_tokens,
33
+ stream=True,
34
+ temperature=temperature,
35
+ top_p=top_p,
36
+ ):
37
+ token = message.choices[0].delta.content
38
+
39
+ response += token
40
+ yield response
41
+
42
+ """
43
+ For information on how to customize the ChatInterface, peruse the gradio docs: https://www.gradio.app/docs/chatinterface
44
+ """
45
+ demo = gr.ChatInterface(
46
+ respond,
47
+ additional_inputs=[
48
+ gr.Textbox(value="You are a friendly Chatbot.", label="System message"),
49
+ gr.Slider(minimum=1, maximum=2048, value=512, step=1, label="Max new tokens"),
50
+ gr.Slider(minimum=0.1, maximum=4.0, value=0.7, step=0.1, label="Temperature"),
51
+ gr.Slider(
52
+ minimum=0.1,
53
+ maximum=1.0,
54
+ value=0.95,
55
+ step=0.05,
56
+ label="Top-p (nucleus sampling)",
57
+ ),
58
+ ],
59
+ )
60
+
61
+
62
+ if __name__ == "__main__":
63
+ demo.launch()