|
import pandas as pd |
|
import numpy as np |
|
|
|
|
|
data = { |
|
"Models": [ |
|
"CodeGen-16B-Multi", |
|
"StarCoder-15B", |
|
"StarCoderBase-15B", |
|
"StarCoderBase-7B", |
|
"StarCoderBase-3B", |
|
"Replit-2.7B", |
|
"SantaCoder-1.1B", |
|
"StarCoderBase-1.1B", |
|
"CodeGen25-7B-mono", |
|
"CodeGen25-7B-multi", |
|
], |
|
"Size (B)": [16, 15, 15, 7, 3, 2.7, 1.1, 1.1, 7, 7], |
|
"Throughput (tokens/s)": [17.20, 38.60, 44.20, 43.10, 50.00, 42.20, 50.80, 71.40, 34.10, 32.60], |
|
"Throughput (tokens/s) bs=50": [0.00, 1490.00, 1460.00, 1700.00, 1770.00, 577.00, 2270.00, 2360.00, 687.00, 680.00], |
|
"Seq_length": [2048, 8192, 8192, 8192, 8192, 2048, 2048, 8192, 2048, 2048], |
|
"#Languages": [6, 86, 86, 86, 86, 20, 3, 86, 86, 86], |
|
"humaneval-python": [19.26, 33.57, 30.35, 28.37, 21.50, 20.12, 18.12, 15.17, 33.08, 28.70], |
|
"java": [22.20, 30.22, 28.53, 24.44, 19.25, 21.39, 15.00, 14.20, 19.75, 26.01], |
|
"javascript": [19.15, 30.79, 31.70, 27.35, 21.32, 20.18, 15.47, 13.38, 23.22, 26.27], |
|
"cpp": [21.00, 31.55, 30.56, 23.30, 19.43, 20.37, 6.20, 11.68, 18.62, 25.75], |
|
"php": [8.37, 26.08, 26.75, 22.12, 18.55, 16.14, 1.50, 9.94, 16.75, 21.98], |
|
"julia": [0.00, 23.02, 21.09, 21.77, 16.10, 1.24, 0.00, 11.31, 4.65, 19.11], |
|
"d": [7.68, 13.57, 10.01, 8.10, 4.97, 6.41, 0.00, 4.65, 4.32, 8.84], |
|
"lua": [8.50, 23.89, 26.61, 23.35, 18.04, 2.11, 0.10, 12.52, 6.75, 23.44], |
|
"r": [6.45, 15.50, 10.18, 14.51, 10.10, 7.20, 0.00, 5.73, 4.41, 11.59], |
|
"ruby": [0.00, 1.24, 17.25, 18.39, 3.93, 10.75, 0.00, 0.31, 0.00, 17.72], |
|
"racket": [0.66, 0.07, 11.77, 11.08, 7.87, 3.22, 0.00, 5.03, 4.07, 10.37], |
|
"rust": [4.21, 21.84, 24.46, 22.60, 16.32, 15.19, 2.00, 10.24, 7.83, 21.84], |
|
"swift": [1.25, 22.74, 16.74, 15.10, 9.98, 5.88, 0.70, 3.92, 1.71, 16.62], |
|
"Peak Memory (MB)": [32890, 33461, 32366, 16512, 8414, 7176, 4602, 4586, 15336, 15336], |
|
} |
|
|
|
df = pd.DataFrame(data).set_index("Models") |
|
df = df.reset_index().rename(columns={"index": "Language"}) |
|
|
|
temp_df = df.copy() |
|
temp_df = temp_df.apply(pd.to_numeric, errors="coerce") |
|
temp_df[temp_df <= 1] = np.nan |
|
|
|
|
|
temp_ = temp_df.iloc[:, 6:-1] |
|
print(temp_) |
|
df.insert(2, "Average score", temp_.mean(axis=1).round(2)) |
|
|
|
print(df["Average score"]) |
|
|
|
|
|
df = df.sort_values(by=["Average score"], ascending=False) |
|
|
|
df.to_csv("/fsx/loubna/code/code-leaderboard/starcoder-models-eval/code_eval_board.csv", index=False) |
|
print(df) |
|
|