Prototype

app.py

@@ -0,0 +1,55 @@
+import gradio as gr
+import pandas as pd
+import numpy as np
+
+# Read the CSV file
+df = pd.read_csv('passk.csv')
+
+# Extract unique languages and models from the Dataset column
+languages = sorted(set([d.split('-')[1] for d in df['Dataset']]))
+models = sorted(set(['-'.join(d.split('-')[2:-2]) for d in df['Dataset']]))
+
+# Create a dictionary to map models to friendly names
+model_to_friendly = {
+    "starcoder2_15b": "StarCoder2-15B",
+    "deepseekcoder_v2lite": "DeepSeekCoder2-Lite"
+}
+
+# Function to get friendly name or original name if not in the dictionary
+def get_friendly_name(model):
+    return model_to_friendly.get(model, model)
+
+# Create a pivot table
+pivot = df.pivot(index='Dataset', columns='Dataset', values='Estimate')
+pivot.index = ['-'.join(i.split('-')[2:-2]) for i in pivot.index]
+pivot.columns = [i.split('-')[1] for i in pivot.columns]
+
+# Function to update the table based on selected languages
+def update_table(selected_languages):
+    if not selected_languages:
+        return np.full((len(models), len(languages)), "-").tolist()
+    
+    display_data = pivot[selected_languages].replace(np.nan, "-")
+    display_data = display_data.applymap(lambda x: f"{x:.3f}" if isinstance(x, (int, float)) else x)
+    display_data.index = [get_friendly_name(model) for model in display_data.index]
+    return display_data.values.tolist()
+
+# Create the Gradio interface
+with gr.Blocks() as app:
+    gr.Markdown("# Model Leaderboard")
+    
+    with gr.Row():
+        language_checkboxes = gr.CheckboxGroup(choices=languages, label="Select Languages", value=languages)
+    
+    table = gr.Dataframe(
+        headers=[lang.capitalize() for lang in languages],
+        row_headers=[get_friendly_name(model) for model in models],
+        col_count=(lambda: len(languages)),
+        interactive=False
+    )
+    
+    language_checkboxes.change(update_table, inputs=[language_checkboxes], outputs=[table])
+
+# Launch the app
+if __name__ == "__main__":
+    app.launch()

passk.csv

@@ -0,0 +1,45 @@
+Dataset,Pass@k,Estimate,NumProblems,MinCompletions,MaxCompletions
+humaneval-clj-deepseekcoder_v2lite_base-0.2-reworded,1,0.16,161,50,50
+humaneval-clj-starcoder2_15b-0.2-reworded,1,0.16,161,50,50
+humaneval-cpp-deepseekcoder_v2lite_base-0.2-reworded,1,0.46,161,50,50
+humaneval-cpp-starcoder2_15b-0.2-reworded,1,0.47,161,50,50
+humaneval-cs-deepseekcoder_v2lite_base-0.2-reworded,1,0.23,158,50,50
+humaneval-cs-starcoder2_15b-0.2-reworded,1,0.32,158,50,50
+humaneval-d-deepseekcoder_v2lite_base-0.2-reworded,1,0.16,156,50,50
+humaneval-d-starcoder2_15b-0.2-reworded,1,0.25,156,50,50
+humaneval-elixir-deepseekcoder_v2lite_base-0.2-reworded,1,0.28,161,50,50
+humaneval-elixir-starcoder2_15b-0.2-reworded,1,0.08,161,50,50
+humaneval-go-deepseekcoder_v2lite_base-0.2-reworded,1,0.28,154,50,50
+humaneval-go-starcoder2_15b-0.2-reworded,1,0.26,154,50,50
+humaneval-hs-deepseekcoder_v2lite_base-0.2-reworded,1,0.20,156,50,50
+humaneval-hs-starcoder2_15b-0.2-reworded,1,0.17,156,50,50
+humaneval-java-deepseekcoder_v2lite_base-0.2-reworded,1,0.36,158,50,50
+humaneval-java-starcoder2_15b-0.2-reworded,1,0.40,158,50,50
+humaneval-jl-deepseekcoder_v2lite_base-0.2-reworded,1,0.33,159,50,50
+humaneval-jl-starcoder2_15b-0.2-reworded,1,0.32,159,50,50
+humaneval-js-deepseekcoder_v2lite_base-0.2-reworded,1,0.45,161,50,50
+humaneval-js-starcoder2_15b-0.2-reworded,1,0.45,161,50,50
+humaneval-lua-deepseekcoder_v2lite_base-0.2-reworded,1,0.38,161,50,50
+humaneval-lua-starcoder2_15b-0.2-reworded,1,0.44,161,50,50
+humaneval-ml-deepseekcoder_v2lite_base-0.2-reworded,1,0.17,155,50,50
+humaneval-ml-starcoder2_15b-0.2-reworded,1,0.24,155,50,50
+humaneval-php-deepseekcoder_v2lite_base-0.2-reworded,1,0.43,161,50,50
+humaneval-php-starcoder2_15b-0.2-reworded,1,0.34,161,50,50
+humaneval-pl-deepseekcoder_v2lite_base-0.2-reworded,1,0.37,161,50,50
+humaneval-pl-starcoder2_15b-0.2-reworded,1,0.38,161,50,50
+humaneval-rb-deepseekcoder_v2lite_base-0.2-reworded,1,0.40,161,50,50
+humaneval-rb-starcoder2_15b-0.2-reworded,1,0.42,161,50,50
+humaneval-r-deepseekcoder_v2lite_base-0.2-reworded,1,0.31,161,50,50
+humaneval-rkt-deepseekcoder_v2lite_base-0.2-reworded,1,0.22,161,50,50
+humaneval-rkt-starcoder2_15b-0.2-reworded,1,0.26,161,50,50
+humaneval-rs-deepseekcoder_v2lite_base-0.2-reworded,1,0.38,156,50,50
+humaneval-rs-starcoder2_15b-0.2-reworded,1,0.39,156,50,50
+humaneval-r-starcoder2_15b-0.2-reworded,1,0.25,161,50,50
+humaneval-scala-deepseekcoder_v2lite_base-0.2-reworded,1,0.39,160,50,50
+humaneval-scala-starcoder2_15b-0.2-reworded,1,0.41,160,50,50
+humaneval-sh-deepseekcoder_v2lite_base-0.2-reworded,1,0.19,158,50,50
+humaneval-sh-starcoder2_15b-0.2-reworded,1,0.19,158,50,50
+humaneval-swift-deepseekcoder_v2lite_base-0.2-reworded,1,0.36,158,50,50
+humaneval-swift-starcoder2_15b-0.2-reworded,1,0.34,158,50,50
+humaneval-ts-deepseekcoder_v2lite_base-0.2-reworded,1,0.47,159,50,50
+humaneval-ts-starcoder2_15b-0.2-reworded,1,0.43,159,50,50

requirements.txt

@@ -0,0 +1,2 @@
+pandas
+numpy