Mark-Arcee commited on
Commit
c0ced35
1 Parent(s): 245cbbb

Upload 3 files

Browse files
Files changed (3) hide show
  1. app.py +128 -0
  2. requirements.txt +8 -0
  3. style.css +17 -0
app.py ADDED
@@ -0,0 +1,128 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #!/usr/bin/env python
2
+
3
+ import os
4
+ from threading import Thread
5
+ from typing import Iterator
6
+
7
+ import gradio as gr
8
+ import spaces
9
+ import torch
10
+ from transformers import AutoModelForCausalLM, AutoTokenizer, TextIteratorStreamer
11
+
12
+ DESCRIPTION = "# Clown-DPO-Extended"
13
+
14
+ if not torch.cuda.is_available():
15
+ DESCRIPTION += "\n<p>Running on CPU 🥶 This demo does not work on CPU.</p>"
16
+
17
+ MAX_MAX_NEW_TOKENS = 2048
18
+ DEFAULT_MAX_NEW_TOKENS = 1024
19
+ MAX_INPUT_TOKEN_LENGTH = int(os.getenv("MAX_INPUT_TOKEN_LENGTH", "4096"))
20
+
21
+ if torch.cuda.is_available():
22
+ model_id = "arcee-ai/Clown-DPO-Extended"
23
+ model = AutoModelForCausalLM.from_pretrained(model_id, torch_dtype=torch.float16, device_map="auto")
24
+ tokenizer = AutoTokenizer.from_pretrained(model_id)
25
+
26
+
27
+ @spaces.GPU
28
+ def generate(
29
+ message: str,
30
+ chat_history: list[tuple[str, str]],
31
+ max_new_tokens: int = 1024,
32
+ temperature: float = 0.6,
33
+ top_p: float = 0.9,
34
+ top_k: int = 50,
35
+ repetition_penalty: float = 1.2,
36
+ ) -> Iterator[str]:
37
+ conversation = []
38
+ for user, assistant in chat_history:
39
+ conversation.extend([{"role": "user", "content": user}, {"role": "assistant", "content": assistant}])
40
+ conversation.append({"role": "user", "content": message})
41
+
42
+ input_ids = tokenizer.apply_chat_template(conversation, return_tensors="pt")
43
+ if input_ids.shape[1] > MAX_INPUT_TOKEN_LENGTH:
44
+ input_ids = input_ids[:, -MAX_INPUT_TOKEN_LENGTH:]
45
+ gr.Warning(f"Trimmed input from conversation as it was longer than {MAX_INPUT_TOKEN_LENGTH} tokens.")
46
+ input_ids = input_ids.to(model.device)
47
+
48
+ streamer = TextIteratorStreamer(tokenizer, timeout=20.0, skip_prompt=True, skip_special_tokens=True)
49
+ generate_kwargs = dict(
50
+ {"input_ids": input_ids},
51
+ streamer=streamer,
52
+ max_new_tokens=max_new_tokens,
53
+ do_sample=True,
54
+ top_p=top_p,
55
+ top_k=top_k,
56
+ temperature=temperature,
57
+ num_beams=1,
58
+ repetition_penalty=repetition_penalty,
59
+ )
60
+ t = Thread(target=model.generate, kwargs=generate_kwargs)
61
+ t.start()
62
+
63
+ outputs = []
64
+ for text in streamer:
65
+ outputs.append(text)
66
+ yield "".join(outputs)
67
+
68
+
69
+ chat_interface = gr.ChatInterface(
70
+ fn=generate,
71
+ additional_inputs=[
72
+ gr.Slider(
73
+ label="Max new tokens",
74
+ minimum=1,
75
+ maximum=MAX_MAX_NEW_TOKENS,
76
+ step=1,
77
+ value=DEFAULT_MAX_NEW_TOKENS,
78
+ ),
79
+ gr.Slider(
80
+ label="Temperature",
81
+ minimum=0.1,
82
+ maximum=4.0,
83
+ step=0.1,
84
+ value=0.6,
85
+ ),
86
+ gr.Slider(
87
+ label="Top-p (nucleus sampling)",
88
+ minimum=0.05,
89
+ maximum=1.0,
90
+ step=0.05,
91
+ value=0.9,
92
+ ),
93
+ gr.Slider(
94
+ label="Top-k",
95
+ minimum=1,
96
+ maximum=1000,
97
+ step=1,
98
+ value=50,
99
+ ),
100
+ gr.Slider(
101
+ label="Repetition penalty",
102
+ minimum=1.0,
103
+ maximum=2.0,
104
+ step=0.05,
105
+ value=1.2,
106
+ ),
107
+ ],
108
+ stop_btn=None,
109
+ examples=[
110
+ ["Hello there! How are you doing?"],
111
+ ["Can you explain briefly to me what is the Python programming language?"],
112
+ ["Explain the plot of Cinderella in a sentence."],
113
+ ["How many hours does it take a man to eat a Helicopter?"],
114
+ ["Write a 100-word article on 'Benefits of Open-Source in AI research'"],
115
+ ],
116
+ )
117
+
118
+ with gr.Blocks(css="style.css") as demo:
119
+ gr.Markdown(DESCRIPTION)
120
+ gr.DuplicateButton(
121
+ value="Duplicate Space for private use",
122
+ elem_id="duplicate-button",
123
+ visible=os.getenv("SHOW_DUPLICATE_BUTTON") == "1",
124
+ )
125
+ chat_interface.render()
126
+
127
+ if __name__ == "__main__":
128
+ demo.queue(max_size=20).launch()
requirements.txt ADDED
@@ -0,0 +1,8 @@
 
 
 
 
 
 
 
 
 
1
+ accelerate==0.27.2
2
+ bitsandbytes==0.42.0
3
+ gradio==4.20.0
4
+ scipy==1.12.0
5
+ sentencepiece==0.1.99
6
+ spaces==0.24.0
7
+ torch==2.0.0
8
+ transformers==4.38.2
style.css ADDED
@@ -0,0 +1,17 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ h1 {
2
+ text-align: center;
3
+ display: block;
4
+ }
5
+
6
+ #duplicate-button {
7
+ margin: auto;
8
+ color: white;
9
+ background: #1565c0;
10
+ border-radius: 100vh;
11
+ }
12
+
13
+ .contain {
14
+ max-width: 900px;
15
+ margin: auto;
16
+ padding-top: 1.5rem;
17
+ }