import gradio as gr
import pandas as pd
from css_html_js import custom_css
demo = gr.Blocks(css=custom_css)
TITLE = """
🤗 Malay LLM Leaderboard
"""
INTRODUCTION_TEXT = """
📐 The 🤗 Malay LLM Leaderboard aims to track, rank and evaluate open LLMs on Malay tasks.\n
🤗 All notebooks at https://github.com/mesolitica/llm-benchmarks, feel free to submit your own score at https://huggingface.co/spaces/mesolitica/malay-llm-leaderboard/discussions with link to the notebook.
## Dataset
📈 We evaluate models based on 4 datasets,
1. BM-PT3 Paper 1, contains 54 questions, https://github.com/mesolitica/malaysian-dataset/tree/master/llm-benchmark/BM-pt3
2. BM Paper 1, contains 180 questions, https://github.com/mesolitica/malaysian-dataset/tree/master/llm-benchmark/tatabahasabm.tripod.com-bm-kertas-1
3. Tatabahasa, contains 349 questions, https://github.com/mesolitica/malaysian-dataset/tree/master/llm-benchmark/tatabahasabm.tripod.com
4. Translated IndoNLI to Malay, tested on `test_expert` dataset, https://huggingface.co/datasets/mesolitica/translated-indonli
"""
data = [
{
'model': 'gpt-3.5-turbo-0613',
'BM-PT3 0-shot (% correct)': 36.53846153846153,
'BM-PT3 1-shot (% correct)': 28.846153846153843,
'BM-PT3 3-shots (% correct)': 24.528301886792452,
},
{
'model': 'malaysian-llama2-7b-32k',
'BM-PT3 0-shot (% correct)': 20.37037037037037,
'BM-PT3 1-shot (% correct)': 16.666666666666664,
'BM-PT3 3-shots (% correct)': 27.77777777777778,
},
{
'model': 'malaysian-llama2-13b-32k',
'BM-PT3 0-shot (% correct)': 33.33333333333333,
'BM-PT3 1-shot (% correct)': 24.074074074074073,
'BM-PT3 3-shots (% correct)': 25.925925925925924,
}
]
data = pd.DataFrame(data)
with demo:
gr.HTML(TITLE)
gr.Markdown(INTRODUCTION_TEXT, elem_classes="markdown-text")
gr.DataFrame(data)
demo.launch()