File size: 9,424 Bytes
be66a58
 
 
 
 
 
2b7c887
be66a58
 
 
5c67b56
be66a58
2268532
2b7c887
be66a58
 
7ce50b1
2b7c887
 
9332ccd
 
be66a58
9332ccd
be66a58
 
 
 
 
 
 
 
 
7ce50b1
be66a58
 
7583b9e
be66a58
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
12f4f82
be66a58
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
9741079
be66a58
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
b0af97b
be66a58
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
import deepsparse
import gradio as gr
from typing import Tuple, List

deepsparse.cpu.print_hardware_capability()

MODEL_ID = "hf:neuralmagic/mpt-7b-gsm8k-pruned60-quant"

DESCRIPTION = f"""
# MPT Sparse Finetuned on GSM8k with DeepSparse 
<img src="https://cdn-uploads.huggingface.co/production/uploads/60466e4b4f40b01b66151416/DOV5q6andhMq83TpAgaU9.jpeg" alt="NM Logo" width="200"/>
Model ID: {MODEL_ID}

πŸš€ **Experience the power of LLM mathematical reasoning** through [our MPT sparse finetuned](https://arxiv.org/abs/2310.06927) on the [GSM8K dataset](https://huggingface.co/datasets/gsm8k). 
GSM8K, short for Grade School Math 8K, is a collection of 8.5K high-quality linguistically diverse grade school math word problems, designed to challenge question-answering systems with multi-step reasoning. 
Observe the model's performance in deciphering complex math questions, such as "Natalia sold clips to 48 of her friends in April, and then she sold half as many clips in May. How many clips did Natalia sell altogether in April and May?" and offering detailed step-by-step solutions.
## Accelerated Inference on CPUs 
The MPT model runs purely on CPU courtesy of [sparse software execution by DeepSparse](https://github.com/neuralmagic/deepsparse/tree/main/research/mpt). 
DeepSparse provides accelerated inference by taking advantage of the MPT model's weight sparsity to deliver tokens fast!

![Speedup](https://cdn-uploads.huggingface.co/production/uploads/60466e4b4f40b01b66151416/qMW-Uq8xAawhANTZYB7ZI.png)
"""

MAX_MAX_NEW_TOKENS = 1024
DEFAULT_MAX_NEW_TOKENS = 200

# Setup the engine
pipe = deepsparse.Pipeline.create(
    task="text-generation",
    model_path=MODEL_ID,
    sequence_length=MAX_MAX_NEW_TOKENS,
    prompt_sequence_length=16,
    num_cores=8,
)

def clear_and_save_textbox(message: str) -> Tuple[str, str]:
    return "", message


def display_input(
    message: str, history: List[Tuple[str, str]]
) -> List[Tuple[str, str]]:
    history.append((message, ""))
    return history


def delete_prev_fn(history: List[Tuple[str, str]]) -> Tuple[List[Tuple[str, str]], str]:
    try:
        message, _ = history.pop()
    except IndexError:
        message = ""
    return history, message or ""
    
with gr.Blocks() as demo:    
    with gr.Row():
        with gr.Column():
            gr.Markdown(DESCRIPTION)
        with gr.Column():
            gr.Markdown("""### MPT GSM Sparse Finetuned Demo""")
            
            with gr.Group():
                chatbot = gr.Chatbot(label="Chatbot")
                with gr.Row():
                    textbox = gr.Textbox(container=False,placeholder="Type a message...",scale=10,)
                    submit_button = gr.Button("Submit", variant="primary", scale=1, min_width=0)
                    
            with gr.Row():
                retry_button = gr.Button("πŸ”„  Retry", variant="secondary")
                undo_button = gr.Button("↩️ Undo", variant="secondary")
                clear_button = gr.Button("πŸ—‘οΈ  Clear", variant="secondary")

            saved_input = gr.State()

            gr.Examples(examples=[
            "James decides to run 3 sprints 3 times a week. He runs 60 meters each sprint. How many total meters does he run a week?",
            "Claire makes a 3 egg omelet every morning for breakfast. How many dozens of eggs will she eat in 4 weeks?",
            "Gretchen has 110 coins. There are 30 more gold coins than silver coins. How many gold coins does Gretchen have?",],inputs=[textbox],)
        
            max_new_tokens = gr.Slider(
                    label="Max new tokens",
                    value=DEFAULT_MAX_NEW_TOKENS,
                    minimum=0,
                    maximum=MAX_MAX_NEW_TOKENS,
                    step=1,
                    interactive=True,
                    info="The maximum numbers of new tokens",)
            temperature = gr.Slider(
                label="Temperature",
                value=0.3,
                minimum=0.05,
                maximum=1.0,
                step=0.05,
                interactive=True,
                info="Higher values produce more diverse outputs",
                            )
            top_p = gr.Slider(
                label="Top-p (nucleus) sampling",
                value=0.40,
                minimum=0.0,
                maximum=1,
                step=0.05,
                interactive=True,
                info="Higher values sample more low-probability tokens",
                            )
            top_k = gr.Slider(
                label="Top-k sampling",
                value=20,
                minimum=1,
                maximum=100,
                step=1,
                interactive=True,
                info="Sample from the top_k most likely tokens",
                )
            repetition_penalty = gr.Slider(
                label="Repetition penalty",
                value=1.2,
                minimum=1.0,
                maximum=2.0,
                step=0.05,
                interactive=True,
                info="Penalize repeated tokens",
                )

            # Generation inference
            def generate(
                        message,
                        history,
                        max_new_tokens: int,
                        temperature: float,
                        top_p: float,
                        top_k: int,
                        repetition_penalty: float,
                ):
                    generation_config = { "max_new_tokens": max_new_tokens,"temperature": temperature,"top_p": top_p,"top_k": top_k,"repetition_penalty": repetition_penalty,}
                    inference = pipe(sequences=message, streaming=True, **generation_config)
                    history[-1][1] += message
                    for token in inference:
                        history[-1][1] += token.generations[0].text
                        yield history
                    print(pipe.timer_manager)
            textbox.submit(
                fn=clear_and_save_textbox,
                inputs=textbox,
                outputs=[textbox, saved_input],
                api_name=False,
                queue=False,
                ).then(
                        fn=display_input,
                        inputs=[saved_input, chatbot],
                        outputs=chatbot,
                        api_name=False,
                        queue=False,
                ).success(
                    generate,
                    inputs=[
                            saved_input,
                            chatbot,
                            max_new_tokens,
                            temperature,
                            top_p,
                            top_k,
                            repetition_penalty,
                    ],
                        outputs=[chatbot],
                        api_name=False,
                    )
                        
            submit_button.click(
                            fn=clear_and_save_textbox,
                            inputs=textbox,
                            outputs=[textbox, saved_input],
                            api_name=False,
                            queue=False,
            ).then(
                            fn=display_input,
                            inputs=[saved_input, chatbot],
                            outputs=chatbot,
                            api_name=False,
                            queue=False,
                ).success(
                            generate,
                            inputs=[saved_input, chatbot, max_new_tokens, temperature],
                            outputs=[chatbot],
                            api_name=False,
                )   
                    
            retry_button.click(
                            fn=delete_prev_fn,
                            inputs=chatbot,
                            outputs=[chatbot, saved_input],
                            api_name=False,
                            queue=False,
            ).then(
                            fn=display_input,
                            inputs=[saved_input, chatbot],
                            outputs=chatbot,
                            api_name=False,
                            queue=False,
            ).then(
                            generate,
                            inputs=[saved_input, chatbot, max_new_tokens, temperature],
                            outputs=[chatbot],
                            api_name=False,
                ) 
            undo_button.click(
                            fn=delete_prev_fn,
                            inputs=chatbot,
                            outputs=[chatbot, saved_input],
                            api_name=False,
                            queue=False,
                ).then(
                            fn=lambda x: x,
                            inputs=[saved_input],
                            outputs=textbox,
                            api_name=False,
                            queue=False,
                    )
            clear_button.click(
                            fn=lambda: ([], ""),
                            outputs=[chatbot, saved_input],
                            queue=False,
                            api_name=False,
                    )
                    
                
            
            
demo.queue().launch()