import gradio as gr
from plotting import create_yolobench_plots, get_pareto_table
from utils import DEEPLITE_DARK_BLUE_GRADIO
def get_hw_description(hw_name):
HW_URLS = {
'Jetson Nano (GPU, ONNX Runtime, FP32)': 'https://8074457.fs1.hubspotusercontent-na1.net/hubfs/8074457/YOLOBench%20Hardware%20product%20sheets/JetsonNano_DataSheet_DS09366001v1.1.pdf',
'Raspberry Pi 4 Model B (CPU, TFLite, FP32)': 'https://8074457.fs1.hubspotusercontent-na1.net/hubfs/8074457/YOLOBench%20Hardware%20product%20sheets/raspberry-pi-4-datasheet.pdf',
'Intel® Core™i7-10875H (CPU, OpenVINO, FP32)': 'https://8074457.fs1.hubspotusercontent-na1.net/hubfs/8074457/YOLOBench%20Hardware%20product%20sheets/Intel_ARK_SpecificationsChart_2023_10_11.pdf',
'Khadas VIM3 (NPU, INT16)': 'https://8074457.fs1.hubspotusercontent-na1.net/hubfs/8074457/YOLOBench%20Hardware%20product%20sheets/khadas_vim3_specs.pdf',
'Orange Pi 5 (NPU, FP16)': 'https://8074457.fs1.hubspotusercontent-na1.net/hubfs/8074457/YOLOBench%20Hardware%20product%20sheets/OrangePi_5_RK3588S_User%20Manual_v1.5.pdf',
}
hw_url = HW_URLS[hw_name]
DESC = f"""
🔸 Click [here]({hw_url}) for more information on the selected hardware platform.
🔸 Refer to the [Deeplite Torch Zoo](https://github.com/Deeplite/deeplite-torch-zoo/tree/develop/results/yolobench) for details about latency measurement experiments.
"""
return DESC
with gr.Blocks(theme=gr.themes.Default(secondary_hue=DEEPLITE_DARK_BLUE_GRADIO),
css="table { width: 100%; }", analytics_enabled=True) as demo:
gr.HTML(
"""
"""
)
# switch to light theme by default
demo.load(
None,
_js="""
() => {
let mediaQueryObj = window.matchMedia('(prefers-color-scheme: dark)');
let systemDarkTheme = window.location.href.includes("theme=system") && mediaQueryObj.matches;
if (window.location.href.includes("theme=dark") || systemDarkTheme){
document.body.classList.toggle('dark');
document.querySelector('gradio-app').style.backgroundColor = 'var(--color-background-primary)'
}
}
""",
)
demo.load(
None,
_js="""
() => {
const script2 = document.createElement("script");
script2.src = "https://www.googletagmanager.com/gtag/js?id=G-01G83VTHE0";
script2.async = true;
document.head.appendChild(script2);
window.dataLayer = window.dataLayer || [];
function gtag(){dataLayer.push(arguments);}
gtag('js', new Date());
gtag('config', 'G-01G83VTHE0', {
'page_path': "/spaces/deepliteai/yolobench",
'page_title': 'yolobench',
'cookie_flags': 'SameSite=None;Secure',
'debug_mode':true,
});
}
""",
)
with gr.Row():
gr.Markdown(
"""
🚀 YOLOBench 🚀 is a latency-accuracy benchmark of popular single-stage detectors from the YOLO series. Major highlights of this work are:
🔸 includes architectures from YOLOv3 to YOLOv8,
🔸 trained on four popular object detection datasets (COCO, VOC, WIDER FACE, SKU-110k),
🔸 latency measured on five embedded hardware platforms (Jetson Nano GPU, ARM CPU, Intel CPU, Khadas VIM3 NPU, Orange Pi NPU),
🔸 all models are trained with the same training loop and hyperparameters (as implemented in the [Ultralytics YOLOv8 codebase](https://github.com/ultralytics/ultralytics)),
🔸 both the detection head structure and the loss function used are that of YOLOv8, giving a chance to isolate the contribution of the backbone/neck architecture on the latency-accuracy trade-off of YOLO models.
In particular, we show that older backbone/neck structures like those of YOLOv3 and YOLOv4 are still competitive compared to more recent architectures in a controlled environment. For more details, please refer to the [arXiv preprint](https://arxiv.org/abs/2307.13901) and the [codebase](https://github.com/Deeplite/deeplite-torch-zoo).
#
#
"""
)
with gr.Row(equal_height=True):
with gr.Column():
hardware_name = gr.Dropdown(
choices=[
'Jetson Nano (GPU, ONNX Runtime, FP32)',
'Raspberry Pi 4 Model B (CPU, TFLite, FP32)',
'Intel® Core™i7-10875H (CPU, OpenVINO, FP32)',
'Khadas VIM3 (NPU, INT16)',
'Orange Pi 5 (NPU, FP16)',
],
value='Jetson Nano (GPU, ONNX Runtime, FP32)',
label='Hardware',
)
with gr.Column():
dataset_name = gr.Dropdown(
choices=['COCO', 'PASCAL VOC', 'SKU-110K', 'WIDERFACE'],
value='COCO',
label='Dataset',
)
with gr.Row(equal_height=True):
with gr.Column():
hardware_desc = gr.Markdown(get_hw_description(hardware_name.value))
with gr.Column():
metric_name = gr.Radio(
['mAP@0.5:0.95', 'mAP@0.5', 'Precision', 'Recall'],
value='mAP@0.5:0.95',
label='Accuracy metric to plot',
)
with gr.Row(equal_height=True):
with gr.Column():
gr.Markdown("""
🚀 Want to add your own hardware benchmarks to YOLOBench? 🚀
Contact us [here](https://info.deeplite.ai/add_yolobench_data) for your benchmarking kit and we'll set you up!
""")
with gr.Column():
vis_options = gr.CheckboxGroup(
[
'Model family',
'Highlight Pareto',
'Show Pareto only',
'Log x-axis'
],
value=['Model family',],
label='Visualization options',
)
with gr.Row():
upper_panel_fig = gr.Plot(show_label=False)
gr.Markdown(
"""
##
Models from this benchmark can be loaded using [Deeplite Torch Zoo](https://github.com/Deeplite/deeplite-torch-zoo) as follows:
##
```python
from deeplite_torch_zoo import get_model
model = get_model(
model_name='yolo4n', # create a YOLOv4n model for the COCO dataset
dataset_name='coco', # (`n` corresponds to width factor 0.25, depth factor 0.33)
pretrained=False, #
custom_head='v8' # attach a YOLOv8 detection head to YOLOv4n backbone+neck
)
```
To train a model, run
```python
from deeplite_torch_zoo.trainer import Detector
model = Detector(torch_model=model) # previously created YOLOv4n model
model.train(data='VOC.yaml', epochs=100, imgsz=480) # same arguments as the Ultralytics trainer object
```
##
Model naming conventions
##
The model naming convention is that a model named `yolo8d67w25` is a YOLOv8 model with a depth factor of 0.67 and width factor of 0.25. Conventional depth/width factor value namings (n, s, m, l models) are used where possible. YOLOv6(s, m, l) models are considered to be different architectures due to differences other than the depth/width factor value. For every architecture, there are 3 variations in depth factor (0.33, 0.67, 1.0) and 4 variations in width factor (0.25, 0.5, 0.75, 1.0), except for YOLOv7 models, for which only width factor variations are considered while depth is fixed.
##
Pareto-optimal models
##
COCO pre-trained models are ready for download. Other models coming soon!
"""
)
table_mode = gr.Radio(
['Show top-10 models', 'Show all'],
value='Show top-10 models',
label='Pareto model table'
)
with gr.Row():
# pareto_table = gr.DataFrame(interactive=False)
pareto_table = gr.HTML()
gr.Markdown(
"""
## Citation
```
Accepted at ICCV 2023 Workshop on Resource-Efficient Deep Learning for Computer Vision (RCV'23)
@article{lazarevich2023yolobench,
title={YOLOBench: Benchmarking Efficient Object Detectors on Embedded Systems},
author={Lazarevich, Ivan and Grimaldi, Matteo and Kumar, Ravish and Mitra, Saptarshi and Khan, Shahrukh and Sah, Sudhakar},
journal={arXiv preprint arXiv:2307.13901},
year={2023}
}
```
"""
)
inputs = [dataset_name, hardware_name, metric_name, vis_options, table_mode]
# plot by default (VOC, Raspi4)
demo.load(
fn=create_yolobench_plots,
inputs=inputs,
outputs=[upper_panel_fig, pareto_table],
)
demo.load(
fn=get_pareto_table,
inputs=[dataset_name, hardware_name, metric_name],
outputs=[pareto_table],
)
# update in case of dataset selection
dataset_name.change(
fn=create_yolobench_plots,
inputs=inputs,
outputs=[upper_panel_fig, pareto_table],
)
# update in case of metric selection
metric_name.change(
fn=create_yolobench_plots,
inputs=inputs,
outputs=[upper_panel_fig, pareto_table],
)
vis_options.change(
fn=create_yolobench_plots,
inputs=inputs,
outputs=[upper_panel_fig, pareto_table],
)
table_mode.change(
fn=create_yolobench_plots,
inputs=inputs,
outputs=[upper_panel_fig, pareto_table],
)
# update in case of device selection
hardware_name.change(
fn=create_yolobench_plots,
inputs=inputs,
outputs=[upper_panel_fig, pareto_table],
)
hardware_name.change(
fn=get_hw_description,
inputs=[hardware_name],
outputs=[hardware_desc],
)
if __name__ == "__main__":
demo.launch()