Spaces:
Running
Running
Update app.py
Browse files
app.py
CHANGED
@@ -1,12 +1,57 @@
|
|
1 |
import gradio as gr
|
2 |
|
3 |
-
class ConversionTool:
|
4 |
-
def __init__(self):
|
5 |
-
# Initialize widgets
|
6 |
-
self.intro = gr.Markdown(INTRODUCTION)
|
7 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
8 |
|
|
|
9 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
10 |
|
11 |
self.model_input = gr.Textbox(
|
12 |
label='Model',
|
@@ -30,7 +75,8 @@ class ConversionTool:
|
|
30 |
'multiple-choice', 'depth-estimation', 'image-classification',
|
31 |
'fill-mask', 'zero-shot-object-detection', 'object-detection',
|
32 |
'question-answering', 'zero-shot-image-classification',
|
33 |
-
'mask-generation', 'text-generation', 'text-classification'
|
|
|
34 |
],
|
35 |
value=None
|
36 |
)
|
@@ -135,38 +181,36 @@ class ConversionTool:
|
|
135 |
"""Construct the command string"""
|
136 |
if not model_input or not output_path:
|
137 |
return ''
|
138 |
-
cmd_parts = ['optimum-cli export openvino']
|
139 |
|
140 |
-
|
|
|
141 |
cmd_parts.append(f'-m "{model_input}"')
|
142 |
-
|
143 |
-
|
144 |
-
# Optional arguments
|
145 |
-
if task != 'auto':
|
146 |
cmd_parts.append(f'--task {task}')
|
147 |
-
|
148 |
-
if framework
|
149 |
cmd_parts.append(f'--framework {framework}')
|
150 |
-
|
151 |
-
if weight_format != 'fp32':
|
152 |
cmd_parts.append(f'--weight-format {weight_format}')
|
153 |
-
|
154 |
-
if library != 'auto':
|
155 |
cmd_parts.append(f'--library {library}')
|
156 |
-
|
157 |
-
if ratio !=
|
158 |
cmd_parts.append(f'--ratio {ratio}')
|
159 |
-
|
160 |
-
if group_size !=
|
161 |
cmd_parts.append(f'--group-size {group_size}')
|
162 |
-
|
163 |
-
if backup_precision
|
164 |
cmd_parts.append(f'--backup-precision {backup_precision}')
|
165 |
-
|
166 |
-
if dataset != 'none':
|
167 |
cmd_parts.append(f'--dataset {dataset}')
|
168 |
-
|
169 |
-
#
|
170 |
if trust_remote_code:
|
171 |
cmd_parts.append('--trust-remote-code')
|
172 |
if disable_stateful:
|
@@ -185,21 +229,24 @@ class ConversionTool:
|
|
185 |
cmd_parts.append('--lora-correction')
|
186 |
if sym:
|
187 |
cmd_parts.append('--sym')
|
188 |
-
|
189 |
-
#
|
190 |
if quant_mode:
|
191 |
cmd_parts.append(f'--quant-mode {quant_mode}')
|
192 |
if cache_dir:
|
193 |
-
cmd_parts.append(f'--cache_dir {cache_dir}')
|
194 |
-
if pad_token_id:
|
195 |
cmd_parts.append(f'--pad-token-id {pad_token_id}')
|
196 |
if sensitivity_metric:
|
197 |
cmd_parts.append(f'--sensitivity-metric {sensitivity_metric}')
|
198 |
-
if num_samples:
|
199 |
cmd_parts.append(f'--num-samples {num_samples}')
|
200 |
-
if smooth_quant_alpha:
|
201 |
cmd_parts.append(f'--smooth-quant-alpha {smooth_quant_alpha}')
|
202 |
-
|
|
|
|
|
|
|
203 |
constructed_command = ' '.join(cmd_parts)
|
204 |
return constructed_command
|
205 |
|
@@ -237,18 +284,13 @@ class ConversionTool:
|
|
237 |
inputs=inputs,
|
238 |
outputs=self.command_output,
|
239 |
title="OpenVINO Conversion Tool",
|
240 |
-
description="Enter
|
|
|
|
|
241 |
)
|
242 |
|
243 |
-
# Add custom CSS to make labels bold
|
244 |
-
interface.css = """
|
245 |
-
label {
|
246 |
-
font-weight: bold !important;
|
247 |
-
}
|
248 |
-
"""
|
249 |
|
250 |
return interface
|
251 |
-
|
252 |
|
253 |
if __name__ == "__main__":
|
254 |
tool = ConversionTool()
|
@@ -257,51 +299,3 @@ if __name__ == "__main__":
|
|
257 |
|
258 |
|
259 |
|
260 |
-
INTRODUCTION="""
|
261 |
-
|
262 |
-
### # Optimum CLI Export Tool.. tool
|
263 |
-
|
264 |
-
This tool helps organize conversion commands when using Intel Optimum for Transformers and respects the order of positional arguments. Otherwise these commands can get quite nuanced to keep track of.
|
265 |
-
|
266 |
-
My goal was to make it easier to construct commands for the [Optimum CLI conversion tool](https://huggingface.co/docs/optimum/main/en/intel/openvino/export) which enables converting models to the OpenVINO Intermediate Representation
|
267 |
-
outside of the from.pretrained method used in Transformers with OpenVINO related classes like OVModelForCausalLM, OVModelForSeq2SeqLM, OVModelForQuestionAnswering, etc, which interface with the OpenVINO runtime.
|
268 |
-
|
269 |
-
## Usage
|
270 |
-
Here I'm assuming you have followed the instructions in the documentation and have all your dependencies in order.
|
271 |
-
|
272 |
-
Run to to get the latest version of the neccessary extension for optimum:
|
273 |
-
```
|
274 |
-
pip install --upgrade --upgrade-strategy eager optimum[openvino]
|
275 |
-
```
|
276 |
-
|
277 |
-
Intended workflow:
|
278 |
-
-Select conversion parameters.
|
279 |
-
-Hit "Submit"
|
280 |
-
-Copy command.
|
281 |
-
-Execute in your environment.
|
282 |
-
|
283 |
-
Note: Converstion can take a while and will be resource intensive.
|
284 |
-
|
285 |
-
|
286 |
-
OpenVINO supports Intel CPUs from 6th gen forward, so you can squeeze performance out of older hardware with
|
287 |
-
different accuracy/performance tradeoffs than the popular quants of GGUFs.
|
288 |
-
|
289 |
-
## Discussion
|
290 |
-
|
291 |
-
Leveraging CPU, GPU and NPU hardware acceleration from OpenVINO requires converting a model into an Intermediate format derived from ONNX.
|
292 |
-
The command we execute rebuilds the model graph from it's source to be optimized for how OpenVINO uses this graph in memory.
|
293 |
-
|
294 |
-
Using OpenVINO effectively requires considering facts about your Intel hardware. Visit the [Intel Ark ]([Intel® Processors for PC, Laptops, Servers, and AI | Intel®](https://www.intel.com/content/www/us/en/products/details/processors.html)) product database to find this information.
|
295 |
-
|
296 |
-
Here are some hardware questions you should be able to answer before using this tool;
|
297 |
-
|
298 |
-
- What data types does my CPU support?
|
299 |
-
- What instruction sets?
|
300 |
-
- How will I be using the model?
|
301 |
-
- Do I have enough system memory for this task?
|
302 |
-
|
303 |
-
|
304 |
-
|
305 |
-
It's *the* ground truth for Intel Hardware specs. Even so, when testing with different model architectures
|
306 |
-
|
307 |
-
"""
|
|
|
1 |
import gradio as gr
|
2 |
|
|
|
|
|
|
|
|
|
3 |
|
4 |
+
INTRODUCTION="""
|
5 |
+
|
6 |
+
### # Optimum CLI Export Tool.. tool
|
7 |
+
|
8 |
+
This tool helps organize conversion commands when using Intel Optimum for Transformers and respects the order of positional arguments. Otherwise these commands can get quite nuanced to keep track of.
|
9 |
+
|
10 |
+
My goal was to make it easier to construct commands for the [Optimum CLI conversion tool](https://huggingface.co/docs/optimum/main/en/intel/openvino/export) which enables converting models to the OpenVINO Intermediate Representation
|
11 |
+
outside of the from.pretrained method used in Transformers with OpenVINO related classes like OVModelForCausalLM, OVModelForSeq2SeqLM, OVModelForQuestionAnswering, etc, which interface with the OpenVINO runtime.
|
12 |
+
|
13 |
+
## Usage
|
14 |
+
Here I'm assuming you have followed the instructions in the documentation and have all your dependencies in order.
|
15 |
+
|
16 |
+
Run to to get the latest version of the neccessary extension for optimum:
|
17 |
+
```
|
18 |
+
pip install --upgrade --upgrade-strategy eager optimum[openvino]
|
19 |
+
```
|
20 |
+
|
21 |
+
Intended workflow:
|
22 |
+
-Select conversion parameters.
|
23 |
+
-Hit "Submit"
|
24 |
+
-Copy command.
|
25 |
+
-Execute in your environment.
|
26 |
+
|
27 |
+
Note: Converstion can take a while and will be resource intensive.
|
28 |
+
|
29 |
+
|
30 |
+
OpenVINO supports Intel CPUs from 6th gen forward, so you can squeeze performance out of older hardware with
|
31 |
+
different accuracy/performance tradeoffs than the popular quants of GGUFs.
|
32 |
+
|
33 |
+
## Discussion
|
34 |
+
|
35 |
+
Leveraging CPU, GPU and NPU hardware acceleration from OpenVINO requires converting a model into an Intermediate format derived from ONNX.
|
36 |
+
The command we execute rebuilds the model graph from it's source to be optimized for how OpenVINO uses this graph in memory.
|
37 |
|
38 |
+
Using OpenVINO effectively requires considering facts about your Intel hardware. Visit the [Intel Ark]([Intel® Processors for PC, Laptops, Servers, and AI | Intel®](https://www.intel.com/content/www/us/en/products/details/processors.html)) product database to find this information.
|
39 |
|
40 |
+
Here are some hardware questions you should be able to answer before using this tool;
|
41 |
+
|
42 |
+
- What data types does my CPU support?
|
43 |
+
- What instruction sets?
|
44 |
+
- How will I be using the model?
|
45 |
+
- Do I have enough system memory for this task?
|
46 |
+
|
47 |
+
|
48 |
+
|
49 |
+
It's *the* ground truth for Intel Hardware specs. Even so, when testing with different model architectures
|
50 |
+
|
51 |
+
"""
|
52 |
+
|
53 |
+
class ConversionTool:
|
54 |
+
def __init__(self):
|
55 |
|
56 |
self.model_input = gr.Textbox(
|
57 |
label='Model',
|
|
|
75 |
'multiple-choice', 'depth-estimation', 'image-classification',
|
76 |
'fill-mask', 'zero-shot-object-detection', 'object-detection',
|
77 |
'question-answering', 'zero-shot-image-classification',
|
78 |
+
'mask-generation', 'text-generation', 'text-classification',
|
79 |
+
'text-to-text-generation', 'text-generation-with-past'
|
80 |
],
|
81 |
value=None
|
82 |
)
|
|
|
181 |
"""Construct the command string"""
|
182 |
if not model_input or not output_path:
|
183 |
return ''
|
|
|
184 |
|
185 |
+
|
186 |
+
cmd_parts = ['optimum-cli export openvino']
|
187 |
cmd_parts.append(f'-m "{model_input}"')
|
188 |
+
|
189 |
+
if task and task != 'auto':
|
|
|
|
|
190 |
cmd_parts.append(f'--task {task}')
|
191 |
+
|
192 |
+
if framework:
|
193 |
cmd_parts.append(f'--framework {framework}')
|
194 |
+
|
195 |
+
if weight_format and weight_format != 'fp32':
|
196 |
cmd_parts.append(f'--weight-format {weight_format}')
|
197 |
+
|
198 |
+
if library and library != 'auto':
|
199 |
cmd_parts.append(f'--library {library}')
|
200 |
+
|
201 |
+
if ratio is not None and ratio != 0:
|
202 |
cmd_parts.append(f'--ratio {ratio}')
|
203 |
+
|
204 |
+
if group_size is not None and group_size != 0:
|
205 |
cmd_parts.append(f'--group-size {group_size}')
|
206 |
+
|
207 |
+
if backup_precision:
|
208 |
cmd_parts.append(f'--backup-precision {backup_precision}')
|
209 |
+
|
210 |
+
if dataset and dataset != 'none':
|
211 |
cmd_parts.append(f'--dataset {dataset}')
|
212 |
+
|
213 |
+
# Boolean flags - only add if True
|
214 |
if trust_remote_code:
|
215 |
cmd_parts.append('--trust-remote-code')
|
216 |
if disable_stateful:
|
|
|
229 |
cmd_parts.append('--lora-correction')
|
230 |
if sym:
|
231 |
cmd_parts.append('--sym')
|
232 |
+
|
233 |
+
# Additional optional arguments - only add if they have values
|
234 |
if quant_mode:
|
235 |
cmd_parts.append(f'--quant-mode {quant_mode}')
|
236 |
if cache_dir:
|
237 |
+
cmd_parts.append(f'--cache_dir "{cache_dir}"')
|
238 |
+
if pad_token_id is not None and pad_token_id != 0:
|
239 |
cmd_parts.append(f'--pad-token-id {pad_token_id}')
|
240 |
if sensitivity_metric:
|
241 |
cmd_parts.append(f'--sensitivity-metric {sensitivity_metric}')
|
242 |
+
if num_samples is not None and num_samples != 0:
|
243 |
cmd_parts.append(f'--num-samples {num_samples}')
|
244 |
+
if smooth_quant_alpha is not None and smooth_quant_alpha != 0:
|
245 |
cmd_parts.append(f'--smooth-quant-alpha {smooth_quant_alpha}')
|
246 |
+
|
247 |
+
cmd_parts.append(f'"{output_path}"')
|
248 |
+
|
249 |
+
|
250 |
constructed_command = ' '.join(cmd_parts)
|
251 |
return constructed_command
|
252 |
|
|
|
284 |
inputs=inputs,
|
285 |
outputs=self.command_output,
|
286 |
title="OpenVINO Conversion Tool",
|
287 |
+
description="Enter model information to generate an `optimum-cli` export command.",
|
288 |
+
article=INTRODUCTION,
|
289 |
+
allow_flagging='auto'
|
290 |
)
|
291 |
|
|
|
|
|
|
|
|
|
|
|
|
|
292 |
|
293 |
return interface
|
|
|
294 |
|
295 |
if __name__ == "__main__":
|
296 |
tool = ConversionTool()
|
|
|
299 |
|
300 |
|
301 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|