| import gradio as gr | |
| import pandas as pd | |
| from css_html_js import custom_css | |
| TITLE = """<h1 align="center" id="space-title">🇲🇾 Malaysian Embedding Leaderboard</h1>""" | |
| INTRODUCTION_TEXT = """ | |
| 📐 The 🇲🇾 Malaysian Embedding Leaderboard aims to track, rank and evaluate Top-k retrieval using embedding models. All notebooks at https://github.com/mesolitica/embedding-benchmarks, feel free to submit your own score at https://huggingface.co/spaces/mesolitica/Malaysian-Embedding-Leaderboard/discussions with link to the notebook. | |
| ## Dataset | |
| 📈 We evaluate models based on 2 datasets, | |
| 1. Research paper keyword `melayu` using Crossref, https://huggingface.co/datasets/mesolitica/malaysian-ultrachat/resolve/main/ultrachat-crossref-melayu-malay.jsonl | |
| 2. Epenerbitan, https://huggingface.co/datasets/mesolitica/malaysian-ultrachat/resolve/main/ultrachat-epenerbitan-malay.jsonl | |
| """ | |
| close_source = [ | |
| { | |
| 'model': 'OpenAI ADA-002', | |
| 'Crossref Melayu top-1': 0.3155939351340496, | |
| 'Crossref Melayu top-3': 0.5120996083944171, | |
| 'Crossref Melayu top-5': 0.5878100210864544, | |
| 'Crossref Melayu top-10': 0.6721558389396526, | |
| 'lom.agc.gov.my top-1': 0.19168533731640527, | |
| 'lom.agc.gov.my top-3': 0.2827981080408265, | |
| 'lom.agc.gov.my top-5': 0.322504356484939, | |
| 'lom.agc.gov.my top-10': 0.36855862584017923, | |
| } | |
| ] | |
| open_source = [ | |
| { | |
| 'model': '[llama2-embedding-600m-8k](https://huggingface.co/mesolitica/llama2-embedding-600m-8k)', | |
| 'Crossref Melayu top-1': 0.09549151521237072, | |
| 'Crossref Melayu top-3': 0.1834521538307059, | |
| 'Crossref Melayu top-5': 0.23375840947886334, | |
| 'Crossref Melayu top-10': 0.3098704689225826, | |
| 'lom.agc.gov.my top-1': 0.05215334826985312, | |
| 'lom.agc.gov.my top-3': 0.09932785660941, | |
| 'lom.agc.gov.my top-5': 0.12969878018421707, | |
| 'lom.agc.gov.my top-10': 0.1797361214836943, | |
| }, | |
| { | |
| 'model': '[llama2-embedding-1b-8k](https://huggingface.co/mesolitica/llama2-embedding-1b-8k)', | |
| 'Crossref Melayu top-1': 0.06777788934631991, | |
| 'Crossref Melayu top-3': 0.142584596847073, | |
| 'Crossref Melayu top-5': 0.18817150316296816, | |
| 'Crossref Melayu top-10': 0.25715433276433375, | |
| 'lom.agc.gov.my top-1': 0.06870799103808813, | |
| 'lom.agc.gov.my top-3': 0.1343042071197411, | |
| 'lom.agc.gov.my top-5': 0.1717699775952203, | |
| 'lom.agc.gov.my top-10': 0.23089370176748816, | |
| }, | |
| { | |
| 'model': '[llama2-embedding-600m-8k-contrastive](https://huggingface.co/mesolitica/llama2-embedding-600m-8k)', | |
| 'Crossref Melayu top-1': 0.11015162164875991, | |
| 'Crossref Melayu top-3': 0.23707199518023897, | |
| 'Crossref Melayu top-5': 0.30916758710713926, | |
| 'Crossref Melayu top-10': 0.4196204438196606, | |
| 'lom.agc.gov.my top-1': 0.05414488424197162, | |
| 'lom.agc.gov.my top-3': 0.11600697037590242, | |
| 'lom.agc.gov.my top-5': 0.16143888473985563, | |
| 'lom.agc.gov.my top-10': 0.23823749066467514, | |
| }, | |
| { | |
| 'model': '[llama2-embedding-1b-8k-contrastive](https://huggingface.co/mesolitica/llama2-embedding-1b-8k)', | |
| 'Crossref Melayu top-1': 0.16306858118284967, | |
| 'Crossref Melayu top-3': 0.32824580781202933, | |
| 'Crossref Melayu top-5': 0.41409780098403454, | |
| 'Crossref Melayu top-10': 0.5312782407872276, | |
| 'lom.agc.gov.my top-1': 0.08824993776450087, | |
| 'lom.agc.gov.my top-3': 0.17836694050286284, | |
| 'lom.agc.gov.my top-5': 0.2399800846402788, | |
| 'lom.agc.gov.my top-10': 0.3343291013193926, | |
| }, | |
| ] | |
| data = pd.DataFrame(close_source + open_source) | |
| demo = gr.Blocks(css=custom_css) | |
| with demo: | |
| gr.HTML(TITLE) | |
| gr.Markdown(INTRODUCTION_TEXT, elem_classes="markdown-text") | |
| gr.DataFrame(data, datatype = 'markdown') | |
| demo.launch() |