Spaces:
Running
Running
| import os | |
| import time | |
| import traceback | |
| from config_store import ( | |
| get_process_config, | |
| get_inference_config, | |
| get_openvino_config, | |
| get_pytorch_config, | |
| ) | |
| import gradio as gr | |
| from huggingface_hub import create_repo, whoami | |
| from optimum_benchmark.launchers.device_isolation_utils import * # noqa | |
| from optimum_benchmark.backends.openvino.utils import TASKS_TO_OVMODEL | |
| from optimum_benchmark.backends.transformers_utils import TASKS_TO_MODEL_LOADERS | |
| from optimum_benchmark import ( | |
| Benchmark, | |
| BenchmarkConfig, | |
| ProcessConfig, | |
| InferenceConfig, | |
| PyTorchConfig, | |
| OVConfig, | |
| ) | |
| from optimum_benchmark.logging_utils import setup_logging | |
| DEVICE = "cpu" | |
| LAUNCHER = "process" | |
| SCENARIO = "inference" | |
| BACKENDS = ["pytorch", "openvino"] | |
| MODELS = [ | |
| "openai-community/gpt2", | |
| "google-bert/bert-base-uncased", | |
| "hf-internal-testing/tiny-random-LlamaForCausalLM", | |
| "hf-internal-testing/tiny-random-BertForSequenceClassification", | |
| ] | |
| MODELS_TO_TASKS = { | |
| "openai-community/gpt2": "text-generation", | |
| "google-bert/bert-base-uncased": "text-classification", | |
| "hf-internal-testing/tiny-random-LlamaForCausalLM": "text-generation", | |
| "hf-internal-testing/tiny-random-BertForSequenceClassification": "text-classification", | |
| } | |
| TASKS = set(TASKS_TO_OVMODEL.keys()) & set(TASKS_TO_MODEL_LOADERS.keys()) | |
| def run_benchmark(kwargs, oauth_token: gr.OAuthToken): | |
| if oauth_token.token is None: | |
| gr.Error("Please login to be able to run the benchmark.") | |
| return tuple(None for _ in BACKENDS) | |
| timestamp = time.strftime("%Y-%m-%d-%H-%M-%S") | |
| username = whoami(oauth_token.token)["name"] | |
| repo_id = f"{username}/benchmarks" | |
| token = oauth_token.token | |
| create_repo(repo_id, token=token, repo_type="dataset", exist_ok=True) | |
| gr.Info(f'Created repository "{repo_id}" where results will be pushed.') | |
| configs = { | |
| "process": {}, | |
| "inference": {}, | |
| "pytorch": {}, | |
| "openvino": {}, | |
| } | |
| for key, value in kwargs.items(): | |
| if key.label == "model": | |
| model = value | |
| elif key.label == "task": | |
| task = value | |
| elif key.label == "backends": | |
| backends = value | |
| elif "." in key.label: | |
| backend, argument = key.label.split(".") | |
| configs[backend][argument] = value | |
| else: | |
| continue | |
| for key in configs.keys(): | |
| for k, v in configs[key].items(): | |
| if k in ["input_shapes", "generate_kwargs", "numactl_kwargs"]: | |
| configs[key][k] = eval(v) | |
| configs["process"] = ProcessConfig(**configs.pop("process")) | |
| configs["inference"] = InferenceConfig(**configs.pop("inference")) | |
| configs["pytorch"] = PyTorchConfig( | |
| task=task, | |
| model=model, | |
| device=DEVICE, | |
| **configs["pytorch"], | |
| ) | |
| configs["openvino"] = OVConfig( | |
| task=task, | |
| model=model, | |
| device=DEVICE, | |
| **configs["openvino"], | |
| ) | |
| outputs = { | |
| "pytorch": "Running benchmark for PyTorch backend", | |
| "openvino": "Running benchmark for OpenVINO backend", | |
| } | |
| yield tuple(outputs[b] for b in BACKENDS) | |
| for backend in backends: | |
| try: | |
| benchmark_name = f"{timestamp}/{backend}" | |
| benchmark_config = BenchmarkConfig( | |
| name=benchmark_name, | |
| backend=configs[backend], | |
| launcher=configs[LAUNCHER], | |
| scenario=configs[SCENARIO], | |
| ) | |
| benchmark_config.push_to_hub( | |
| repo_id=repo_id, subfolder=benchmark_name, token=oauth_token.token | |
| ) | |
| benchmark_report = Benchmark.launch(benchmark_config) | |
| benchmark_report.push_to_hub( | |
| repo_id=repo_id, subfolder=benchmark_name, token=oauth_token.token | |
| ) | |
| benchmark = Benchmark(config=benchmark_config, report=benchmark_report) | |
| benchmark.push_to_hub( | |
| repo_id=repo_id, subfolder=benchmark_name, token=oauth_token.token | |
| ) | |
| gr.Info(f"Pushed benchmark to {username}/benchmarks/{benchmark_name}") | |
| outputs[backend] = f"\n{benchmark_report.to_markdown_text()}" | |
| yield tuple(outputs[b] for b in BACKENDS) | |
| except Exception: | |
| gr.Error(f"Error while running benchmark for {backend}") | |
| outputs[backend] = f"\n```python\n{traceback.format_exc()}```" | |
| yield tuple(outputs[b] for b in BACKENDS) | |
| def build_demo(): | |
| with gr.Blocks() as demo: | |
| # add login button | |
| gr.LoginButton(min_width=250) | |
| # add image | |
| gr.HTML( | |
| """<img src="https://huggingface.co/spaces/optimum/optimum-benchmark-ui/resolve/main/huggy_bench.png" style="display: block; margin-left: auto; margin-right: auto; width: 30%;">""" | |
| "<h1 style='text-align: center'>🤗 Optimum-Benchmark Interface 🏋️</h1>" | |
| "<p style='text-align: center'>" | |
| "This Space uses <a href='https://github.com/huggingface/optimum-benchmark.git'>Optimum-Benchmark</a> to automatically benchmark a model from the Hub on different backends." | |
| "<br>The results (config and report) will be pushed under your namespace in a benchmark repository on the Hub." | |
| "</p>" | |
| ) | |
| model = gr.Dropdown( | |
| label="model", | |
| choices=MODELS, | |
| value=MODELS[0], | |
| info="Model to run the benchmark on.", | |
| ) | |
| task = gr.Dropdown( | |
| label="task", | |
| choices=TASKS, | |
| value="feature-extraction", | |
| info="Task to run the benchmark on.", | |
| ) | |
| backends = gr.CheckboxGroup( | |
| interactive=True, | |
| label="backends", | |
| choices=BACKENDS, | |
| value=BACKENDS, | |
| info="Backends to run the benchmark on.", | |
| ) | |
| with gr.Row(): | |
| with gr.Accordion(label="Process Config", open=False, visible=True): | |
| process_config = get_process_config() | |
| with gr.Row(): | |
| with gr.Accordion(label="Inference Config", open=False, visible=True): | |
| inference_config = get_inference_config() | |
| with gr.Row() as backend_configs: | |
| with gr.Accordion(label="PyTorch Config", open=False, visible=True): | |
| pytorch_config = get_pytorch_config() | |
| with gr.Accordion(label="OpenVINO Config", open=False, visible=True): | |
| openvino_config = get_openvino_config() | |
| with gr.Row(): | |
| button = gr.Button(value="Run Benchmark", variant="primary") | |
| with gr.Row() as markdown_outputs: | |
| with gr.Accordion(label="PyTorch Output", open=True, visible=True): | |
| pytorch_output = gr.Markdown() | |
| with gr.Accordion(label="OpenVINO Output", open=True, visible=True): | |
| openvino_output = gr.Markdown() | |
| model.change( | |
| inputs=model, outputs=task, fn=lambda value: MODELS_TO_TASKS[value] | |
| ) | |
| backends.change( | |
| inputs=backends, | |
| outputs=backend_configs.children, | |
| fn=lambda values: [ | |
| gr.update(visible=value in values) for value in BACKENDS | |
| ], | |
| ) | |
| backends.change( | |
| inputs=backends, | |
| outputs=markdown_outputs.children, | |
| fn=lambda values: [ | |
| gr.update(visible=value in values) for value in BACKENDS | |
| ], | |
| ) | |
| button.click( | |
| fn=run_benchmark, | |
| inputs={ | |
| task, | |
| model, | |
| backends, | |
| *process_config.values(), | |
| *inference_config.values(), | |
| *pytorch_config.values(), | |
| *openvino_config.values(), | |
| }, | |
| outputs={ | |
| pytorch_output, | |
| openvino_output, | |
| }, | |
| concurrency_limit=1, | |
| ) | |
| return demo | |
| demo = build_demo() | |
| if __name__ == "__main__": | |
| os.environ["LOG_TO_FILE"] = "0" | |
| os.environ["LOG_LEVEL"] = "INFO" | |
| setup_logging(level="INFO", prefix="MAIN-PROCESS") | |
| demo.queue(max_size=10).launch() | |