File size: 2,257 Bytes
80c91d0 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 |
import plotly.graph_objects as go
import plotly.express as px
import numpy as np
def plot_elo_mle(df):
fig = px.scatter(df, x="model", y="rating", error_y="error_y",
error_y_minus="error_y_minus",
# title="Bootstrap of Elo MLE Estimates (BigCodeBench-Complete)"
)
fig.update_layout(xaxis_title="Model",
yaxis_title="Rating",
autosize=True,
# width=1300,
# height=900,
)
return fig
def plot_solve_rate(df, task, rows=30, cols=38):
keys = df["task_id"]
values = df["solve_rate"]
values = np.array(values, dtype=float) # Ensure values are floats
# Extract numerical IDs and sort by them
ids = [int(key.split('/')[-1]) for key in keys]
sorted_indices = np.argsort(ids)
keys = np.array(keys)[sorted_indices]
values = values[sorted_indices]
n = len(values)
pad_width = rows * cols - n
# Create a masked array
masked_values = np.ma.array(np.full(rows * cols, np.nan), mask=True)
masked_values[:n] = values
masked_values.mask[:n] = False
masked_values = masked_values.reshape((rows, cols))
keys_padded = np.pad(keys, (0, pad_width), 'constant', constant_values='')
keys_reshaped = keys_padded.reshape((rows, cols))
hover_text = np.empty_like(masked_values, dtype=object)
for i in range(rows):
for j in range(cols):
if not masked_values.mask[i, j]:
hover_text[i, j] = f"{keys_reshaped[i, j]}<br>Solve Rate: {masked_values[i, j]:.2f}"
else:
hover_text[i, j] = "NaN"
upper_solve_rate = round(np.count_nonzero(values) / n * 100, 2)
fig = go.Figure(data=go.Heatmap(
z=masked_values,
text=hover_text,
hoverinfo='text',
colorscale='teal',
zmin=0,
zmax=100
))
fig.update_layout(
title=f'BigCodeBench-{task}<br><i>Lowest Upper Limit: {upper_solve_rate}%</i>',
xaxis_nticks=cols,
yaxis_nticks=rows,
xaxis=dict(showticklabels=False),
yaxis=dict(showticklabels=False),
autosize=True,
)
return fig |