loubnabnl's picture
loubnabnl HF staff
update data
d5c587b
raw
history blame
2.56 kB
import pandas as pd
import numpy as np
data = {
"Models": [
"CodeGen-16B-Multi",
"StarCoder-15B",
"StarCoderBase-15B",
"StarCoderBase-7B",
"StarCoderBase-3B",
"Replit-2.7B",
"SantaCoder-1.1B",
"StarCoderBase-1.1B",
"CodeGen25-7B-mono",
"CodeGen25-7B-multi",
],
"Size (B)": [16, 15, 15, 7, 3, 2.7, 1.1, 1.1, 7, 7],
"Throughput (tokens/s)": [17.20, 38.60, 44.20, 43.10, 50.00, 42.20, 50.80, 71.40, 34.10, 32.60],
"Throughput (tokens/s) bs=50": [0.00, 1490.00, 1460.00, 1700.00, 1770.00, 577.00, 2270.00, 2360.00, 687.00, 680.00],
"Seq_length": [2048, 8192, 8192, 8192, 8192, 2048, 2048, 8192, 2048, 2048],
"#Languages": [6, 86, 86, 86, 86, 20, 3, 86, 86, 86],
"humaneval-python": [19.26, 33.57, 30.35, 28.37, 21.50, 20.12, 18.12, 15.17, 33.08, 28.70],
"java": [22.20, 30.22, 28.53, 24.44, 19.25, 21.39, 15.00, 14.20, 19.75, 26.01],
"javascript": [19.15, 30.79, 31.70, 27.35, 21.32, 20.18, 15.47, 13.38, 23.22, 26.27],
"cpp": [21.00, 31.55, 30.56, 23.30, 19.43, 20.37, 6.20, 11.68, 18.62, 25.75],
"php": [8.37, 26.08, 26.75, 22.12, 18.55, 16.14, 1.50, 9.94, 16.75, 21.98],
"julia": [0.00, 23.02, 21.09, 21.77, 16.10, 1.24, 0.00, 11.31, 4.65, 19.11],
"d": [7.68, 13.57, 10.01, 8.10, 4.97, 6.41, 0.00, 4.65, 4.32, 8.84],
"lua": [8.50, 23.89, 26.61, 23.35, 18.04, 2.11, 0.10, 12.52, 6.75, 23.44],
"r": [6.45, 15.50, 10.18, 14.51, 10.10, 7.20, 0.00, 5.73, 4.41, 11.59],
"ruby": [0.00, 1.24, 17.25, 18.39, 3.93, 10.75, 0.00, 0.31, 0.00, 17.72],
"racket": [0.66, 0.07, 11.77, 11.08, 7.87, 3.22, 0.00, 5.03, 4.07, 10.37],
"rust": [4.21, 21.84, 24.46, 22.60, 16.32, 15.19, 2.00, 10.24, 7.83, 21.84],
"swift": [1.25, 22.74, 16.74, 15.10, 9.98, 5.88, 0.70, 3.92, 1.71, 16.62],
"Peak Memory (MB)": [32890, 33461, 32366, 16512, 8414, 7176, 4602, 4586, 15336, 15336],
}
df = pd.DataFrame(data).set_index("Models")
df = df.reset_index().rename(columns={"index": "Language"})
temp_df = df.copy()
temp_df = temp_df.apply(pd.to_numeric, errors="coerce")
temp_df[temp_df <= 1] = np.nan
# get average over all columns from index 4 until -1 not included
temp_ = temp_df.iloc[:, 6:-1]
print(temp_)
df.insert(2, "Average score", temp_.mean(axis=1).round(2))
#print average score
print(df["Average score"])
# sort with regard to column average
df = df.sort_values(by=["Average score"], ascending=False)
df.to_csv("/fsx/loubna/code/code-leaderboard/starcoder-models-eval/code_eval_board.csv", index=False)
print(df)