added more test set

app.py

@@ -2,17 +2,21 @@ import gradio as gr
 import pandas as pd
 from css_html_js import custom_css
 
-TITLE = """<h1 align="center" id="space-title">🇲🇾 Malaysian RAG Embedding Leaderboard</h1>"""
+TITLE = """<h1 align="center" id="space-title">🇲🇾 Malaysian Embedding Leaderboard</h1>"""
 
 INTRODUCTION_TEXT = """
-📐 The 🇲🇾 Malaysian RAG Embedding Leaderboard aims to track, rank and evaluate Top-k retrieval using embedding models. All notebooks at https://github.com/mesolitica/embedding-benchmarks, feel free to submit your own score at https://huggingface.co/spaces/mesolitica/Malaysian-Embedding-Leaderboard/discussions with link to the notebook.
+📐 The 🇲🇾 Malaysian Embedding Leaderboard aims to track, rank and evaluate Top-k retrieval using embedding models. All notebooks at https://github.com/mesolitica/embedding-benchmarks, feel free to submit your own score at https://huggingface.co/spaces/mesolitica/Malaysian-Embedding-Leaderboard/discussions with link to the notebook.
 
 ## Dataset
 
 📈 We evaluate models based on 2 datasets,
 
-1. Research paper keyword `melayu` using Crossref, https://huggingface.co/datasets/mesolitica/malaysian-ultrachat/resolve/main/ultrachat-crossref-melayu-malay.jsonl
-2. lom.agc.gov.my PDF files, https://huggingface.co/datasets/mesolitica/malaysian-ultrachat/resolve/main/ultrachat-lom-agc.jsonl
+1. Research paper keyword `melayu` using Crossref, **this test set is about RAG**, https://huggingface.co/datasets/mesolitica/malaysian-ultrachat/resolve/main/ultrachat-crossref-melayu-malay.jsonl
+2. lom.agc.gov.my PDF files, **this test set is about RAG**, https://huggingface.co/datasets/mesolitica/malaysian-ultrachat/resolve/main/ultrachat-lom-agc.jsonl
+3. b.cari.com.my, **this test set is about semantic similarity**, https://huggingface.co/datasets/mesolitica/embedding-pair-mining/resolve/main/b.cari.com.my-dataset-bge-test.jsonl
+3. c.cari.com.my, **this test set is about semantic similarity**, https://huggingface.co/datasets/mesolitica/embedding-pair-mining/resolve/main/c.cari.com.my-dataset-bge-test.jsonl
+3. Malay news, **this test set is about semantic similarity**, https://huggingface.co/datasets/mesolitica/embedding-pair-mining/resolve/main/malay-news-dataset-bge-test.jsonl
+3. Twitter, **this test set is about semantic similarity**, https://huggingface.co/datasets/mesolitica/embedding-pair-mining/resolve/main/twitter-dataset-bge-test.jsonl
 """
 
 close_source = [
@@ -26,6 +30,22 @@ close_source = [
         'lom.agc.gov.my top-3': 0.2827981080408265,
         'lom.agc.gov.my top-5': 0.322504356484939,
         'lom.agc.gov.my top-10': 0.36855862584017923,
+        'b.cari.com.my top-1': 0.31621790857858484,
+        'b.cari.com.my top-3': 0.6242955541640576,
+        'b.cari.com.my top-5': 0.6944270507201001,
+        'b.cari.com.my top-10': 0.7623669380087664,
+        'c.cari.com.my top-1': 0.08380430943129637,
+        'c.cari.com.my top-3': 0.21388202048746027,
+        'c.cari.com.my top-5': 0.27861179795125396,
+        'c.cari.com.my top-10': 0.3589720946661957,
+        'malay-news top-1': 0.1386895659334196,
+        'malay-news top-3': 0.2952593812492648,
+        'malay-news top-5': 0.3745441712739678,
+        'malay-news top-10': 0.4754734737089754,
+        'twitter top-1': 0.22942090082094518,
+        'twitter top-3': 0.4919014865764367,
+        'twitter top-5': 0.5930774351009541,
+        'twitter top-10': 0.7248724206789439,
     }
 ]
 
@@ -40,6 +60,22 @@ open_source = [
         'lom.agc.gov.my top-3': 0.09932785660941,
         'lom.agc.gov.my top-5': 0.12969878018421707,
         'lom.agc.gov.my top-10': 0.1797361214836943,
+        'b.cari.com.my top-1': 0.3168440826549781,
+        'b.cari.com.my top-3': 0.6881653099561679,
+        'b.cari.com.my top-5': 0.7789605510331872,
+        'b.cari.com.my top-10': 0.8453350031308704,
+        'c.cari.com.my top-1': 0.08327446132108796,
+        'c.cari.com.my top-3': 0.18730130695867184,
+        'c.cari.com.my top-5': 0.23975626986930412,
+        'c.cari.com.my top-10': 0.3140233133168492,
+        'malay-news top-1': 0.14268909540054112,
+        'malay-news top-3': 0.27584990001176335,
+        'malay-news top-5': 0.3640748147276791,
+        'malay-news top-10': 0.47112104458299026,
+        'twitter top-1': 0.23496782782338585,
+        'twitter top-3': 0.5200798757488352,
+        'twitter top-5': 0.6416685156423342,
+        'twitter top-10': 0.785888617705791,
     },
     {
         'model': '[mesolitica/llama2-embedding-1b-8k](https://huggingface.co/mesolitica/llama2-embedding-1b-8k)',
@@ -51,6 +87,22 @@ open_source = [
         'lom.agc.gov.my top-3': 0.1343042071197411,
         'lom.agc.gov.my top-5': 0.1717699775952203,
         'lom.agc.gov.my top-10': 0.23089370176748816,
+        'b.cari.com.my top-1': 0.32623669380087666,
+        'b.cari.com.my top-3': 0.6947401377582968,
+        'b.cari.com.my top-5': 0.7902316844082655,
+        'b.cari.com.my top-10': 0.8603631809643081,
+        'c.cari.com.my top-1': 0.08689509007417874,
+        'c.cari.com.my top-3': 0.19966442953020133,
+        'c.cari.com.my top-5': 0.26086188625927237,
+        'c.cari.com.my top-10': 0.3430766513599435,
+        'malay-news top-1': 0.14610045876955652,
+        'malay-news top-3': 0.285260557581461,
+        'malay-news top-5': 0.3640748147276791,
+        'malay-news top-10': 0.4775908716621574,
+        'twitter top-1': 0.23053028622143332,
+        'twitter top-3': 0.5342800088750832,
+        'twitter top-5': 0.6498779676059463,
+        'twitter top-10': 0.7925449301087197,
     },
     {
         'model': '[mesolitica/mistral-embedding-191m-8k-contrastive](https://huggingface.co/mesolitica/mistral-embedding-191m-8k-contrastive)',
@@ -62,6 +114,22 @@ open_source = [
         'lom.agc.gov.my top-3': 0.09148618371919343,
         'lom.agc.gov.my top-5': 0.12758277321384118,
         'lom.agc.gov.my top-10': 0.18707991038088126,
+        'b.cari.com.my top-1': 0.2564182842830307,
+        'b.cari.com.my top-3': 0.46900438321853477,
+        'b.cari.com.my top-5': 0.5309956167814652,
+        'b.cari.com.my top-10': 0.6036318096430808,
+        'c.cari.com.my top-1': 0.05298481102084069,
+        'c.cari.com.my top-3': 0.10005298481102085,
+        'c.cari.com.my top-5': 0.12433768986223949,
+        'c.cari.com.my top-10': 0.16990462734016248,
+        'malay-news top-1': 0.1102223267850841,
+        'malay-news top-3': 0.1672744383013763,
+        'malay-news top-5': 0.20091753911304552,
+        'malay-news top-10': 0.24973532525585226,
+        'twitter top-1': 0.18437985356112713,
+        'twitter top-3': 0.3536720656756157,
+        'twitter top-5': 0.422897714666075,
+        'twitter top-10': 0.5227424007100067,
     },
     {
         'model': '[mesolitica/mistral-embedding-349m-8k-contrastive](https://huggingface.co/mesolitica/mistral-embedding-349m-8k-contrastive)',
@@ -73,6 +141,22 @@ open_source = [
         'lom.agc.gov.my top-3': 0.08849887976101568,
         'lom.agc.gov.my top-5': 0.12335075927308937,
         'lom.agc.gov.my top-10': 0.18558625840179238,
+        'b.cari.com.my top-1': 0.24953036944270507,
+        'b.cari.com.my top-3': 0.4730745147150908,
+        'b.cari.com.my top-5': 0.5341264871634315,
+        'b.cari.com.my top-10': 0.5970569818409518,
+        'c.cari.com.my top-1': 0.057400211939244085,
+        'c.cari.com.my top-3': 0.10588131402331331,
+        'c.cari.com.my top-5': 0.13149063935005298,
+        'c.cari.com.my top-10': 0.17582126457082303,
+        'malay-news top-1': 0.11375132337372074,
+        'malay-news top-3': 0.17362663216092225,
+        'malay-news top-5': 0.20574050111751557,
+        'malay-news top-10': 0.26149864721797433,
+        'twitter top-1': 0.18105169735966276,
+        'twitter top-3': 0.3547814510761038,
+        'twitter top-5': 0.42666962502773464,
+        'twitter top-10': 0.5229642777901042,
     },
     {
         'model': '[mesolitica/llama2-embedding-600m-8k-contrastive](https://huggingface.co/mesolitica/llama2-embedding-600m-8k-contrastive)',
@@ -84,6 +168,22 @@ open_source = [
         'lom.agc.gov.my top-3': 0.13754045307443366,
         'lom.agc.gov.my top-5': 0.18869803335822755,
         'lom.agc.gov.my top-10': 0.2745830221558377,
+        'b.cari.com.my top-1': 0.254539762053851,
+        'b.cari.com.my top-3': 0.4815278647463995,
+        'b.cari.com.my top-5': 0.5497808390732624,
+        'b.cari.com.my top-10': 0.628052598622417,
+        'c.cari.com.my top-1': 0.05183680678205581,
+        'c.cari.com.my top-3': 0.09766866831508302,
+        'c.cari.com.my top-5': 0.12398445778876722,
+        'c.cari.com.my top-10': 0.16566584245849522,
+        'malay-news top-1': 0.11657452064463004,
+        'malay-news top-3': 0.18986001646865075,
+        'malay-news top-5': 0.22338548406069875,
+        'malay-news top-10': 0.2812610281143395,
+        'twitter top-1': 0.1961393388063013,
+        'twitter top-3': 0.393831817173286,
+        'twitter top-5': 0.48080763257155534,
+        'twitter top-10': 0.5977368537830042,
     },
     {
         'model': '[mesolitica/llama2-embedding-1b-8k-contrastive](https://huggingface.co/mesolitica/llama2-embedding-1b-8k-contrastive)',
@@ -95,6 +195,22 @@ open_source = [
         'lom.agc.gov.my top-3': 0.239731142643764,
         'lom.agc.gov.my top-5': 0.3094349016679114,
         'lom.agc.gov.my top-10': 0.4134926562111028,
+        'b.cari.com.my top-1': 0.29805886036318097,
+        'b.cari.com.my top-3': 0.5795241077019412,
+        'b.cari.com.my top-5': 0.6581089542892924,
+        'b.cari.com.my top-10': 0.7366938008766437,
+        'c.cari.com.my top-1': 0.06720240197809961,
+        'c.cari.com.my top-3': 0.13678912045213706,
+        'c.cari.com.my top-5': 0.1750264924055104,
+        'c.cari.com.my top-10': 0.22818791946308725,
+        'malay-news top-1': 0.12892600870485826,
+        'malay-news top-3': 0.20785789907069757,
+        'malay-news top-5': 0.24808846018115516,
+        'malay-news top-10': 0.32466768615457003,
+        'twitter top-1': 0.2167739072553805,
+        'twitter top-3': 0.4453072997559352,
+        'twitter top-5': 0.5387175504770357,
+        'twitter top-10': 0.6685156423341468,
     },
     {
         'model': '[mesolitica/llama2-embedding-2b-8k-contrastive](https://huggingface.co/mesolitica/llama2-embedding-2b-8k-contrastive)',
@@ -106,6 +222,22 @@ open_source = [
         'lom.agc.gov.my top-3': 0.29113766492407267,
         'lom.agc.gov.my top-5': 0.3725416977844162,
         'lom.agc.gov.my top-10': 0.4886731391585761,
+        'b.cari.com.my top-1': 0.30713838447088293,
+        'b.cari.com.my top-3': 0.6361928616155291,
+        'b.cari.com.my top-5': 0.7213525360050094,
+        'b.cari.com.my top-10': 0.8015028177833438,
+        'c.cari.com.my top-1': 0.07444365948428117,
+        'c.cari.com.my top-3': 0.15745319675026492,
+        'c.cari.com.my top-5': 0.19931119745672907,
+        'c.cari.com.my top-10': 0.26960438007771104,
+        'malay-news top-1': 0.13633690154099518,
+        'malay-news top-3': 0.23056111045759323,
+        'malay-news top-5': 0.28726032231502174,
+        'malay-news top-10': 0.3717209740030585,
+        'twitter top-1': 0.22498335921899268,
+        'twitter top-3': 0.48679831373419125,
+        'twitter top-5': 0.5850898602174396,
+        'twitter top-10': 0.7333037497226537,
     },
 ]