import gradio as gr import pandas as pd from css_html_js import custom_css TITLE = """

🇲🇾 Malaysian RAG Embedding Leaderboard

""" INTRODUCTION_TEXT = """ 📐 The 🇲🇾 Malaysian RAG Embedding Leaderboard aims to track, rank and evaluate Top-k retrieval using embedding models. All notebooks at https://github.com/mesolitica/embedding-benchmarks, feel free to submit your own score at https://huggingface.co/spaces/mesolitica/Malaysian-Embedding-Leaderboard/discussions with link to the notebook. ## Dataset 📈 We evaluate models based on 2 datasets, 1. Research paper keyword `melayu` using Crossref, https://huggingface.co/datasets/mesolitica/malaysian-ultrachat/resolve/main/ultrachat-crossref-melayu-malay.jsonl 2. lom.agc.gov.my PDF files, https://huggingface.co/datasets/mesolitica/malaysian-ultrachat/resolve/main/ultrachat-lom-agc.jsonl """ close_source = [ { 'model': 'OpenAI ADA-002', 'Crossref Melayu top-1': 0.3155939351340496, 'Crossref Melayu top-3': 0.5120996083944171, 'Crossref Melayu top-5': 0.5878100210864544, 'Crossref Melayu top-10': 0.6721558389396526, 'lom.agc.gov.my top-1': 0.19168533731640527, 'lom.agc.gov.my top-3': 0.2827981080408265, 'lom.agc.gov.my top-5': 0.322504356484939, 'lom.agc.gov.my top-10': 0.36855862584017923, } ] open_source = [ { 'model': '[mesolitica/llama2-embedding-600m-8k](https://huggingface.co/mesolitica/llama2-embedding-600m-8k)', 'Crossref Melayu top-1': 0.09549151521237072, 'Crossref Melayu top-3': 0.1834521538307059, 'Crossref Melayu top-5': 0.23375840947886334, 'Crossref Melayu top-10': 0.3098704689225826, 'lom.agc.gov.my top-1': 0.05215334826985312, 'lom.agc.gov.my top-3': 0.09932785660941, 'lom.agc.gov.my top-5': 0.12969878018421707, 'lom.agc.gov.my top-10': 0.1797361214836943, }, { 'model': '[mesolitica/llama2-embedding-1b-8k](https://huggingface.co/mesolitica/llama2-embedding-1b-8k)', 'Crossref Melayu top-1': 0.06777788934631991, 'Crossref Melayu top-3': 0.142584596847073, 'Crossref Melayu top-5': 0.18817150316296816, 'Crossref Melayu top-10': 0.25715433276433375, 'lom.agc.gov.my top-1': 0.06870799103808813, 'lom.agc.gov.my top-3': 0.1343042071197411, 'lom.agc.gov.my top-5': 0.1717699775952203, 'lom.agc.gov.my top-10': 0.23089370176748816, }, { 'model': '[mesolitica/mistral-embedding-191m-8k-contrastive](https://huggingface.co/mesolitica/mistral-embedding-191m-8k-contrastive)', 'Crossref Melayu top-1': 0.08001654088700506, 'Crossref Melayu top-3': 0.17378269409697095, 'Crossref Melayu top-5': 0.232192701333609, 'Crossref Melayu top-10': 0.32482166856197664, 'lom.agc.gov.my top-1': 0.041075429424943986, 'lom.agc.gov.my top-3': 0.09148618371919343, 'lom.agc.gov.my top-5': 0.12758277321384118, 'lom.agc.gov.my top-10': 0.18707991038088126, }, { 'model': '[mesolitica/mistral-embedding-349m-8k-contrastive](https://huggingface.co/mesolitica/mistral-embedding-349m-8k-contrastive)', 'Crossref Melayu top-1': 0.09045797580895276, 'Crossref Melayu top-3': 0.18742892587615012, 'Crossref Melayu top-5': 0.2444949860436266, 'Crossref Melayu top-10': 0.3398118474103174, 'lom.agc.gov.my top-1': 0.039581777445855115, 'lom.agc.gov.my top-3': 0.08849887976101568, 'lom.agc.gov.my top-5': 0.12335075927308937, 'lom.agc.gov.my top-10': 0.18558625840179238, }, { 'model': '[mesolitica/llama2-embedding-600m-8k-contrastive](https://huggingface.co/mesolitica/llama2-embedding-600m-8k)', 'Crossref Melayu top-1': 0.11650987284193115, 'Crossref Melayu top-3': 0.25235190737103275, 'Crossref Melayu top-5': 0.340225369585444, 'Crossref Melayu top-10': 0.45632172025224854, 'lom.agc.gov.my top-1': 0.059870550161812294, 'lom.agc.gov.my top-3': 0.12683594722429675, 'lom.agc.gov.my top-5': 0.1696539706248444, 'lom.agc.gov.my top-10': 0.2534229524520787, }, { 'model': '[mesolitica/llama2-embedding-1b-8k-contrastive](https://huggingface.co/mesolitica/llama2-embedding-1b-8k)', 'Crossref Melayu top-1': 0.21958027499224647, 'Crossref Melayu top-3': 0.42809883179985525, 'Crossref Melayu top-5': 0.5379923498397602, 'Crossref Melayu top-10': 0.6708363485991936, 'lom.agc.gov.my top-1': 0.11899427433408016, 'lom.agc.gov.my top-3': 0.239731142643764, 'lom.agc.gov.my top-5': 0.3094349016679114, 'lom.agc.gov.my top-10': 0.4134926562111028, }, { 'model': '[mesolitica/llama2-embedding-2b-8k-contrastive](https://huggingface.co/mesolitica/llama2-embedding-2b-8k)', 'Crossref Melayu top-1': 0.21399772562803682, 'Crossref Melayu top-3': 0.4175540163341259, 'Crossref Melayu top-5': 0.523415693166546, 'Crossref Melayu top-10': 0.6479892484234467, 'lom.agc.gov.my top-1': 0.12048792631316903, 'lom.agc.gov.my top-3': 0.2455812795618621, 'lom.agc.gov.my top-5': 0.3075678366940503, 'lom.agc.gov.my top-10': 0.4144884241971621, }, ] data = pd.DataFrame(close_source + open_source) demo = gr.Blocks(css=custom_css) with demo: gr.HTML(TITLE) gr.Markdown(INTRODUCTION_TEXT, elem_classes="markdown-text") gr.DataFrame(data, datatype = 'markdown') demo.launch()