| import gradio as gr | |
| import pandas as pd | |
| from css_html_js import custom_css | |
| TITLE = """<h1 align="center" id="space-title">🇲🇾 Malaysian Embedding Leaderboard</h1>""" | |
| INTRODUCTION_TEXT = """ | |
| 📐 The 🇲🇾 Malaysian Embedding Leaderboard aims to track, rank and evaluate Top-k retrieval using embedding models. All notebooks at https://github.com/mesolitica/embedding-benchmarks, feel free to submit your own score at https://huggingface.co/spaces/mesolitica/Malaysian-Embedding-Leaderboard/discussions with link to the notebook. | |
| ## Dataset | |
| 📈 We evaluate models based on 6 datasets, | |
| 1. Research paper keyword `melayu` using Crossref, **this test set is about RAG**, https://huggingface.co/datasets/mesolitica/malaysian-ultrachat/resolve/main/ultrachat-crossref-melayu-malay.jsonl | |
| 2. lom.agc.gov.my PDF files, **this test set is about RAG**, https://huggingface.co/datasets/mesolitica/malaysian-ultrachat/resolve/main/ultrachat-lom-agc.jsonl | |
| 3. b.cari.com.my, **this test set is about semantic similarity**, https://huggingface.co/datasets/mesolitica/embedding-pair-mining/resolve/main/b.cari.com.my-dataset-bge-test.jsonl | |
| 4. c.cari.com.my, **this test set is about semantic similarity**, https://huggingface.co/datasets/mesolitica/embedding-pair-mining/resolve/main/c.cari.com.my-dataset-bge-test.jsonl | |
| 5. Malay news, **this test set is about semantic similarity**, https://huggingface.co/datasets/mesolitica/embedding-pair-mining/resolve/main/malay-news-dataset-bge-test.jsonl | |
| 6. Twitter, **this test set is about semantic similarity**, https://huggingface.co/datasets/mesolitica/embedding-pair-mining/resolve/main/twitter-dataset-bge-test.jsonl | |
| """ | |
| close_source = [ | |
| { | |
| 'model': 'OpenAI ADA-002', | |
| 'Crossref Melayu top-1': 0.3155939351340496, | |
| 'Crossref Melayu top-3': 0.5120996083944171, | |
| 'Crossref Melayu top-5': 0.5878100210864544, | |
| 'Crossref Melayu top-10': 0.6721558389396526, | |
| 'lom.agc.gov.my top-1': 0.19168533731640527, | |
| 'lom.agc.gov.my top-3': 0.2827981080408265, | |
| 'lom.agc.gov.my top-5': 0.322504356484939, | |
| 'lom.agc.gov.my top-10': 0.36855862584017923, | |
| 'b.cari.com.my top-1': 0.31621790857858484, | |
| 'b.cari.com.my top-3': 0.6242955541640576, | |
| 'b.cari.com.my top-5': 0.6944270507201001, | |
| 'b.cari.com.my top-10': 0.7623669380087664, | |
| 'c.cari.com.my top-1': 0.08380430943129637, | |
| 'c.cari.com.my top-3': 0.21388202048746027, | |
| 'c.cari.com.my top-5': 0.27861179795125396, | |
| 'c.cari.com.my top-10': 0.3589720946661957, | |
| 'malay-news top-1': 0.1386895659334196, | |
| 'malay-news top-3': 0.2952593812492648, | |
| 'malay-news top-5': 0.3745441712739678, | |
| 'malay-news top-10': 0.4754734737089754, | |
| 'twitter top-1': 0.22942090082094518, | |
| 'twitter top-3': 0.4919014865764367, | |
| 'twitter top-5': 0.5930774351009541, | |
| 'twitter top-10': 0.7248724206789439, | |
| } | |
| ] | |
| open_source = [ | |
| { | |
| 'model': '[mesolitica/llama2-embedding-600m-8k](https://huggingface.co/mesolitica/llama2-embedding-600m-8k)', | |
| 'Crossref Melayu top-1': 0.09549151521237072, | |
| 'Crossref Melayu top-3': 0.1834521538307059, | |
| 'Crossref Melayu top-5': 0.23375840947886334, | |
| 'Crossref Melayu top-10': 0.3098704689225826, | |
| 'lom.agc.gov.my top-1': 0.05215334826985312, | |
| 'lom.agc.gov.my top-3': 0.09932785660941, | |
| 'lom.agc.gov.my top-5': 0.12969878018421707, | |
| 'lom.agc.gov.my top-10': 0.1797361214836943, | |
| 'b.cari.com.my top-1': 0.3168440826549781, | |
| 'b.cari.com.my top-3': 0.6881653099561679, | |
| 'b.cari.com.my top-5': 0.7789605510331872, | |
| 'b.cari.com.my top-10': 0.8453350031308704, | |
| 'c.cari.com.my top-1': 0.08327446132108796, | |
| 'c.cari.com.my top-3': 0.18730130695867184, | |
| 'c.cari.com.my top-5': 0.23975626986930412, | |
| 'c.cari.com.my top-10': 0.3140233133168492, | |
| 'malay-news top-1': 0.14268909540054112, | |
| 'malay-news top-3': 0.27584990001176335, | |
| 'malay-news top-5': 0.3640748147276791, | |
| 'malay-news top-10': 0.47112104458299026, | |
| 'twitter top-1': 0.23496782782338585, | |
| 'twitter top-3': 0.5200798757488352, | |
| 'twitter top-5': 0.6416685156423342, | |
| 'twitter top-10': 0.785888617705791, | |
| }, | |
| { | |
| 'model': '[mesolitica/llama2-embedding-1b-8k](https://huggingface.co/mesolitica/llama2-embedding-1b-8k)', | |
| 'Crossref Melayu top-1': 0.06777788934631991, | |
| 'Crossref Melayu top-3': 0.142584596847073, | |
| 'Crossref Melayu top-5': 0.18817150316296816, | |
| 'Crossref Melayu top-10': 0.25715433276433375, | |
| 'lom.agc.gov.my top-1': 0.06870799103808813, | |
| 'lom.agc.gov.my top-3': 0.1343042071197411, | |
| 'lom.agc.gov.my top-5': 0.1717699775952203, | |
| 'lom.agc.gov.my top-10': 0.23089370176748816, | |
| 'b.cari.com.my top-1': 0.32623669380087666, | |
| 'b.cari.com.my top-3': 0.6947401377582968, | |
| 'b.cari.com.my top-5': 0.7902316844082655, | |
| 'b.cari.com.my top-10': 0.8603631809643081, | |
| 'c.cari.com.my top-1': 0.08689509007417874, | |
| 'c.cari.com.my top-3': 0.19966442953020133, | |
| 'c.cari.com.my top-5': 0.26086188625927237, | |
| 'c.cari.com.my top-10': 0.3430766513599435, | |
| 'malay-news top-1': 0.14610045876955652, | |
| 'malay-news top-3': 0.285260557581461, | |
| 'malay-news top-5': 0.3640748147276791, | |
| 'malay-news top-10': 0.4775908716621574, | |
| 'twitter top-1': 0.23053028622143332, | |
| 'twitter top-3': 0.5342800088750832, | |
| 'twitter top-5': 0.6498779676059463, | |
| 'twitter top-10': 0.7925449301087197, | |
| }, | |
| { | |
| 'model': '[mesolitica/mistral-embedding-191m-8k-contrastive](https://huggingface.co/mesolitica/mistral-embedding-191m-8k-contrastive)', | |
| 'Crossref Melayu top-1': 0.08001654088700506, | |
| 'Crossref Melayu top-3': 0.17378269409697095, | |
| 'Crossref Melayu top-5': 0.232192701333609, | |
| 'Crossref Melayu top-10': 0.32482166856197664, | |
| 'lom.agc.gov.my top-1': 0.041075429424943986, | |
| 'lom.agc.gov.my top-3': 0.09148618371919343, | |
| 'lom.agc.gov.my top-5': 0.12758277321384118, | |
| 'lom.agc.gov.my top-10': 0.18707991038088126, | |
| 'b.cari.com.my top-1': 0.2564182842830307, | |
| 'b.cari.com.my top-3': 0.46900438321853477, | |
| 'b.cari.com.my top-5': 0.5309956167814652, | |
| 'b.cari.com.my top-10': 0.6036318096430808, | |
| 'c.cari.com.my top-1': 0.05298481102084069, | |
| 'c.cari.com.my top-3': 0.10005298481102085, | |
| 'c.cari.com.my top-5': 0.12433768986223949, | |
| 'c.cari.com.my top-10': 0.16990462734016248, | |
| 'malay-news top-1': 0.1102223267850841, | |
| 'malay-news top-3': 0.1672744383013763, | |
| 'malay-news top-5': 0.20091753911304552, | |
| 'malay-news top-10': 0.24973532525585226, | |
| 'twitter top-1': 0.18437985356112713, | |
| 'twitter top-3': 0.3536720656756157, | |
| 'twitter top-5': 0.422897714666075, | |
| 'twitter top-10': 0.5227424007100067, | |
| }, | |
| { | |
| 'model': '[mesolitica/mistral-embedding-349m-8k-contrastive](https://huggingface.co/mesolitica/mistral-embedding-349m-8k-contrastive)', | |
| 'Crossref Melayu top-1': 0.09045797580895276, | |
| 'Crossref Melayu top-3': 0.18742892587615012, | |
| 'Crossref Melayu top-5': 0.2444949860436266, | |
| 'Crossref Melayu top-10': 0.3398118474103174, | |
| 'lom.agc.gov.my top-1': 0.039581777445855115, | |
| 'lom.agc.gov.my top-3': 0.08849887976101568, | |
| 'lom.agc.gov.my top-5': 0.12335075927308937, | |
| 'lom.agc.gov.my top-10': 0.18558625840179238, | |
| 'b.cari.com.my top-1': 0.24953036944270507, | |
| 'b.cari.com.my top-3': 0.4730745147150908, | |
| 'b.cari.com.my top-5': 0.5341264871634315, | |
| 'b.cari.com.my top-10': 0.5970569818409518, | |
| 'c.cari.com.my top-1': 0.057400211939244085, | |
| 'c.cari.com.my top-3': 0.10588131402331331, | |
| 'c.cari.com.my top-5': 0.13149063935005298, | |
| 'c.cari.com.my top-10': 0.17582126457082303, | |
| 'malay-news top-1': 0.11375132337372074, | |
| 'malay-news top-3': 0.17362663216092225, | |
| 'malay-news top-5': 0.20574050111751557, | |
| 'malay-news top-10': 0.26149864721797433, | |
| 'twitter top-1': 0.18105169735966276, | |
| 'twitter top-3': 0.3547814510761038, | |
| 'twitter top-5': 0.42666962502773464, | |
| 'twitter top-10': 0.5229642777901042, | |
| }, | |
| { | |
| 'model': '[mesolitica/llama2-embedding-600m-8k-contrastive](https://huggingface.co/mesolitica/llama2-embedding-600m-8k-contrastive)', | |
| 'Crossref Melayu top-1': 0.1260208828698439, | |
| 'Crossref Melayu top-3': 0.2649643337123953, | |
| 'Crossref Melayu top-5': 0.3535614597332782, | |
| 'Crossref Melayu top-10': 0.4751369792205107, | |
| 'lom.agc.gov.my top-1': 0.0660941000746826, | |
| 'lom.agc.gov.my top-3': 0.13754045307443366, | |
| 'lom.agc.gov.my top-5': 0.18869803335822755, | |
| 'lom.agc.gov.my top-10': 0.2745830221558377, | |
| 'b.cari.com.my top-1': 0.254539762053851, | |
| 'b.cari.com.my top-3': 0.4815278647463995, | |
| 'b.cari.com.my top-5': 0.5497808390732624, | |
| 'b.cari.com.my top-10': 0.628052598622417, | |
| 'c.cari.com.my top-1': 0.05183680678205581, | |
| 'c.cari.com.my top-3': 0.09766866831508302, | |
| 'c.cari.com.my top-5': 0.12398445778876722, | |
| 'c.cari.com.my top-10': 0.16566584245849522, | |
| 'malay-news top-1': 0.11657452064463004, | |
| 'malay-news top-3': 0.18986001646865075, | |
| 'malay-news top-5': 0.22338548406069875, | |
| 'malay-news top-10': 0.2812610281143395, | |
| 'twitter top-1': 0.1961393388063013, | |
| 'twitter top-3': 0.393831817173286, | |
| 'twitter top-5': 0.48080763257155534, | |
| 'twitter top-10': 0.5977368537830042, | |
| }, | |
| { | |
| 'model': '[mesolitica/llama2-embedding-1b-8k-contrastive](https://huggingface.co/mesolitica/llama2-embedding-1b-8k-contrastive)', | |
| 'Crossref Melayu top-1': 0.21958027499224647, | |
| 'Crossref Melayu top-3': 0.42809883179985525, | |
| 'Crossref Melayu top-5': 0.5379923498397602, | |
| 'Crossref Melayu top-10': 0.6708363485991936, | |
| 'lom.agc.gov.my top-1': 0.11899427433408016, | |
| 'lom.agc.gov.my top-3': 0.239731142643764, | |
| 'lom.agc.gov.my top-5': 0.3094349016679114, | |
| 'lom.agc.gov.my top-10': 0.4134926562111028, | |
| 'b.cari.com.my top-1': 0.29805886036318097, | |
| 'b.cari.com.my top-3': 0.5795241077019412, | |
| 'b.cari.com.my top-5': 0.6581089542892924, | |
| 'b.cari.com.my top-10': 0.7366938008766437, | |
| 'c.cari.com.my top-1': 0.06720240197809961, | |
| 'c.cari.com.my top-3': 0.13678912045213706, | |
| 'c.cari.com.my top-5': 0.1750264924055104, | |
| 'c.cari.com.my top-10': 0.22818791946308725, | |
| 'malay-news top-1': 0.12892600870485826, | |
| 'malay-news top-3': 0.20785789907069757, | |
| 'malay-news top-5': 0.24808846018115516, | |
| 'malay-news top-10': 0.32466768615457003, | |
| 'twitter top-1': 0.2167739072553805, | |
| 'twitter top-3': 0.4453072997559352, | |
| 'twitter top-5': 0.5387175504770357, | |
| 'twitter top-10': 0.6685156423341468, | |
| }, | |
| { | |
| 'model': '[mesolitica/llama2-embedding-2b-8k-contrastive](https://huggingface.co/mesolitica/llama2-embedding-2b-8k-contrastive)', | |
| 'Crossref Melayu top-1': 0.2671353251318102, | |
| 'Crossref Melayu top-3': 0.49116096350666805, | |
| 'Crossref Melayu top-5': 0.6091181639615424, | |
| 'Crossref Melayu top-10': 0.739791171301561, | |
| 'lom.agc.gov.my top-1': 0.1549663928304705, | |
| 'lom.agc.gov.my top-3': 0.29113766492407267, | |
| 'lom.agc.gov.my top-5': 0.3725416977844162, | |
| 'lom.agc.gov.my top-10': 0.4886731391585761, | |
| 'b.cari.com.my top-1': 0.30713838447088293, | |
| 'b.cari.com.my top-3': 0.6361928616155291, | |
| 'b.cari.com.my top-5': 0.7213525360050094, | |
| 'b.cari.com.my top-10': 0.8015028177833438, | |
| 'c.cari.com.my top-1': 0.07444365948428117, | |
| 'c.cari.com.my top-3': 0.15745319675026492, | |
| 'c.cari.com.my top-5': 0.19931119745672907, | |
| 'c.cari.com.my top-10': 0.26960438007771104, | |
| 'malay-news top-1': 0.13633690154099518, | |
| 'malay-news top-3': 0.23056111045759323, | |
| 'malay-news top-5': 0.28726032231502174, | |
| 'malay-news top-10': 0.3717209740030585, | |
| 'twitter top-1': 0.22498335921899268, | |
| 'twitter top-3': 0.48679831373419125, | |
| 'twitter top-5': 0.5850898602174396, | |
| 'twitter top-10': 0.7333037497226537, | |
| }, | |
| { | |
| 'model': '[BAAI/bge-m3](https://huggingface.co/BAAI/bge-m3)', | |
| 'Crossref Melayu top-1': 0.41331541403907784, | |
| 'Crossref Melayu top-3': 0.669802543161377, | |
| 'Crossref Melayu top-5': 0.7698749095420242, | |
| 'Crossref Melayu top-10': 0.8632275405768635, | |
| 'lom.agc.gov.my top-1': 0.35001244709982576, | |
| 'lom.agc.gov.my top-3': 0.5311177495643515, | |
| 'lom.agc.gov.my top-5': 0.6011949215832711, | |
| 'lom.agc.gov.my top-10': 0.6928055763007219, | |
| 'b.cari.com.my top-1': 0.3306199123356293, | |
| 'b.cari.com.my top-3': 0.7025673137132122, | |
| 'b.cari.com.my top-5': 0.7930494677520351, | |
| 'b.cari.com.my top-10': 0.8647463994990607, | |
| 'c.cari.com.my top-1': 0.08795478629459555, | |
| 'c.cari.com.my top-3': 0.23754856941010244, | |
| 'c.cari.com.my top-5': 0.33574708583539387, | |
| 'c.cari.com.my top-10': 0.4558459908159661, | |
| 'malay-news top-1': 0.1459828255499353, | |
| 'malay-news top-3': 0.3185507587342666, | |
| 'malay-news top-5': 0.4232443241971533, | |
| 'malay-news top-10': 0.5473473708975415, | |
| 'twitter top-1': 0.22919902374084758, | |
| 'twitter top-3': 0.5225205236299091, | |
| 'twitter top-5': 0.6443310406035057, | |
| 'twitter top-10': 0.7836698469048148, | |
| } | |
| ] | |
| data = pd.DataFrame(close_source + open_source) | |
| demo = gr.Blocks(css=custom_css) | |
| with demo: | |
| gr.HTML(TITLE) | |
| gr.Markdown(INTRODUCTION_TEXT, elem_classes="markdown-text") | |
| gr.DataFrame(data, datatype = 'markdown') | |
| demo.launch() |