import plotly.express as px
import plotly.graph_objects as go
from utils import DEEPLITE_LIGHT_BLUE_HEX, load_yolobench_data
df, pareto_indices = load_yolobench_data()
METRIC_NAME_MAPPING = {
'mAP@0.5': 'mAP_0.5',
'mAP@0.5:0.95': 'mAP_0.5:0.95',
'Precision': 'precision',
'Recall': 'recall',
}
METRIC_KEYS_TO_NAMES = {v: k for k, v in METRIC_NAME_MAPPING.items()}
LATENCY_KEYS = {
'Raspberry Pi 4 Model B (CPU, TFLite, FP32)': 'raspi4_tflite_latency',
'Raspberry Pi 4 Model B (CPU, ONNX Runtime, FP32)': 'pi4_ort_latency',
'Raspberry Pi 5 Model B (CPU, ONNX Runtime, FP32)': 'pi5_ort_latency',
'Jetson Nano (GPU, ONNX Runtime, FP32)': 'nano_gpu_latency',
'Intel® Core™i7-10875H (CPU, OpenVINO, FP32)': 'openvino_latency',
'Khadas VIM3 (NPU, INT16)': 'vim3_latency',
'Orange Pi 5 (NPU, FP16)': 'orange_pi_latency',
'NVIDIA A40 (GPU, TensorRT, FP32)': 'a40_trt_latency',
}
LATENCY_KEYS_TO_NAMES = {v: k for k, v in LATENCY_KEYS.items()}
DATASET_TAGS = {
'PASCAL VOC': 'voc',
'SKU-110K': 'sku',
'WIDERFACE': 'wider',
'COCO': 'coco',
}
DATASET_TAGS_TO_NAMES = {v: k for k, v in DATASET_TAGS.items()}
def get_scatter_plot(
dataset_tag,
metric_tag,
latency_key,
model_family_coloring=True,
add_pareto_frontier=False,
plot_pareto_only=False,
log_axis=False,
):
fig_opts, layout_opts = {
'opacity': 0.5,
'color_discrete_sequence': [DEEPLITE_LIGHT_BLUE_HEX],
}, {}
if model_family_coloring:
fig_opts = {
'color': 'model_family',
'opacity': 0.75,
'color_discrete_sequence': px.colors.qualitative.Plotly,
}
layout_opts = {
'legend': dict(
title='Model family
(click to toggle)',
)
}
frontier = None
if plot_pareto_only:
metric_key = f'{metric_tag}_{dataset_tag}'
frontier = pareto_indices[metric_key][latency_key]
fig = px.scatter(
df if frontier is None else df.iloc[frontier, :],
x=latency_key,
y=f'{metric_tag}_{dataset_tag}',
title=f'{METRIC_KEYS_TO_NAMES[metric_tag]}-latency scatter plot',
hover_data={
'model_name': True,
'model_family': False,
latency_key: ':.2f',
f'{metric_tag}_{dataset_tag}': ':.2f',
},
labels={
'model_name': 'Model name',
latency_key: 'Latency',
f'{metric_tag}_{dataset_tag}': METRIC_KEYS_TO_NAMES[metric_tag],
},
template='plotly_white',
**fig_opts,
)
if log_axis:
fig.update_xaxes(type='log')
fig.update_layout(
height=600,
modebar_remove=[
'lasso',
'autoscale',
'zoomin',
'zoomout',
'select2d',
'select',
],
xaxis_title=f'{LATENCY_KEYS_TO_NAMES[latency_key]} latency, ms',
yaxis_title=f"{METRIC_KEYS_TO_NAMES[metric_tag]}",
xaxis=dict(
rangeslider=dict(
visible=True,
bgcolor=DEEPLITE_LIGHT_BLUE_HEX,
thickness=0.02,
),
),
yaxis=dict(
fixedrange=False,
),
hoverlabel=dict(
# bgcolor="white",
font_size=14,
font_family='Source Sans Pro',
),
**layout_opts,
)
if add_pareto_frontier:
fig = pareto_frontier_layer(fig, dataset_tag, metric_tag, latency_key)
fig.update_layout(autosize=True)
return fig
def get_comparison_plot(
dataset_tag,
metric_tag,
latency_keys,
log_axis=False,
remove_marker=False,
):
if len(latency_keys) == 0:
layout_opts = {
"annotations": [
{
"text": "Please select at least 2 hardware targets to compare!",
"showarrow": False,
"font": {"size": 28},
}
]
}
else:
layout_opts = {
'legend': dict(
title='Hardware targets selected
(click to toggle)',
)
}
fig = go.Figure(
layout=go.Layout(
title=go.layout.Title(
text=f'{METRIC_KEYS_TO_NAMES[metric_tag]}-latency Pareto Optimal Models'
)
)
)
if remove_marker:
mode = 'lines'
else:
mode = 'lines+markers'
for latency_key in latency_keys:
metric_key = f'{metric_tag}_{dataset_tag}'
frontier = pareto_indices[metric_key][latency_key]
df_pareto = df.iloc[frontier, :]
model_name = df_pareto['model_name']
fig.add_trace(
go.Scatter(
x=df_pareto[latency_key],
y=df_pareto[f'{metric_tag}_{dataset_tag}'],
name=LATENCY_KEYS_TO_NAMES[latency_key],
mode=mode,
customdata=model_name,
hovertemplate='model name:%{customdata}
latency:%{x:.2f}
metric: %{y:.2f} ',
)
)
if log_axis:
fig.update_xaxes(type='log')
x_axis_title = f'Inference latency, ms (BS=1, log scale)'
else:
x_axis_title = f'Inference latency, ms (BS=1)'
fig.update_layout(
height=600,
plot_bgcolor='white',
modebar_remove=[
'lasso',
'autoscale',
'zoomin',
'zoomout',
'select2d',
'select',
],
xaxis_title=x_axis_title,
yaxis_title=f"{METRIC_KEYS_TO_NAMES[metric_tag]}",
xaxis=dict(
rangeslider=dict(
visible=True,
bgcolor=DEEPLITE_LIGHT_BLUE_HEX,
thickness=0.02,
),
),
yaxis=dict(
fixedrange=False,
),
hoverlabel=dict(
# bgcolor="white",
font_size=14,
font_family='Source Sans Pro',
),
**layout_opts,
)
fig.update_layout(autosize=True, template="plotly_white")
return fig
def pareto_frontier_layer(
fig,
dataset_tag,
metric_tag,
latency_key,
):
metric_key = f'{metric_tag}_{dataset_tag}'
frontier = pareto_indices[metric_key][latency_key]
fig.add_trace(
go.Scatter(
x=df.iloc[frontier, :][latency_key],
y=df.iloc[frontier, :][metric_key],
mode='lines+markers',
opacity=0.5,
line=go.scatter.Line(color='grey'),
showlegend=False,
name=metric_key,
marker=dict(symbol=['circle']),
)
)
return fig
def create_yolobench_plots(
dataset_name,
hardware_name,
metric_name,
vis_options,
table_mode,
):
model_family_coloring = 'Model family' in vis_options
add_pareto_frontier = 'Highlight Pareto' in vis_options
plot_pareto_only = 'Show Pareto only' in vis_options
log_axis = 'Log x-axis' in vis_options
fig = get_scatter_plot(
DATASET_TAGS[dataset_name],
METRIC_NAME_MAPPING[metric_name],
LATENCY_KEYS[hardware_name],
model_family_coloring,
add_pareto_frontier,
plot_pareto_only,
log_axis,
)
pareto_table = get_pareto_table(
dataset_name, hardware_name, metric_name, expand_table='Show all' in table_mode
)
return fig, pareto_table
def pareto_frontier_layer(
fig,
dataset_tag,
metric_tag,
latency_key,
):
metric_key = f'{metric_tag}_{dataset_tag}'
frontier = pareto_indices[metric_key][latency_key]
fig.add_trace(
go.Scatter(
x=df.iloc[frontier, :][latency_key],
y=df.iloc[frontier, :][metric_key],
mode='lines',
opacity=0.5,
line=go.scatter.Line(color='grey'),
showlegend=False,
name=metric_key,
)
)
return fig
def create_comparison_plot(dataset_name, hardware_list, metric_name, vis_options):
log_axis = 'Log x-axis' in vis_options
remove_marker = 'Remove datapoint markers' in vis_options
latency_keys = []
for hardware_name in hardware_list:
latency_keys.append(LATENCY_KEYS[hardware_name])
fig = get_comparison_plot(
DATASET_TAGS[dataset_name],
METRIC_NAME_MAPPING[metric_name],
latency_keys,
log_axis,
remove_marker,
)
return fig
def get_pareto_table(
dataset_name,
hardware_name,
metric_name,
expand_table=False,
):
dataset_tag = DATASET_TAGS[dataset_name]
metric_tag = METRIC_NAME_MAPPING[metric_name]
latency_key = LATENCY_KEYS[hardware_name]
metric_key = f'{metric_tag}_{dataset_tag}'
latency_key_final = f'{LATENCY_KEYS_TO_NAMES[latency_key]} latency, ms'
metric_key_final = METRIC_KEYS_TO_NAMES[metric_tag]
frontier = pareto_indices[metric_key][latency_key]
table_df = df.iloc[frontier, :][['model_name', metric_key, latency_key]]
table_df['Input resolution (px)'] = table_df['model_name'].apply(
lambda name: name.split('_')[-1]
)
table_df['Model name'] = table_df['model_name'].apply(
lambda name: name.split('_')[0]
)
table_df[metric_key_final] = table_df[metric_key].apply(lambda val: round(val, 3))
table_df[latency_key_final] = table_df[latency_key].apply(lambda val: round(val, 2))
def make_clickable(url, name):
return f'{name}'
if dataset_name == 'COCO':
table_df['Download link'] = table_df['model_name'].apply(
lambda name: f'https://download.deeplite.ai/zoo/models/YOLOBench/{name.split("_")[0]}_640.pt'
)
table_df['Download link'] = table_df.apply(
lambda x: make_clickable(x['Download link'], 'Weights download'), axis=1
)
else:
table_df['Download link'] = table_df['model_name'].apply(
lambda s: 'Coming soon'
)
table_df = table_df[
[
'Model name',
'Input resolution (px)',
metric_key_final,
latency_key_final,
'Download link',
]
].sort_values(by=metric_key_final, ascending=False)
if not expand_table:
table_df = table_df.iloc[:10, :]
table_df = table_df.to_html(
classes='table', escape=False, render_links=True, index=False
)
return table_df