import gradio as gr
import pandas as pd
from css_html_js import custom_css

TITLE = """<h1 align="center" id="space-title">🇲🇾 Malaysian Embedding Leaderboard</h1>"""

INTRODUCTION_TEXT = """
📐 The 🇲🇾 Malaysian Embedding Leaderboard aims to track, rank and evaluate Top-k retrieval using embedding models. All notebooks at https://github.com/mesolitica/embedding-benchmarks, feel free to submit your own score at https://huggingface.co/spaces/mesolitica/Malaysian-Embedding-Leaderboard/discussions with link to the notebook.

## Dataset

📈 We evaluate models based on 6 datasets,

1. Research paper keyword `melayu` using Crossref, **this test set is about RAG**, https://huggingface.co/datasets/mesolitica/malaysian-ultrachat/resolve/main/ultrachat-crossref-melayu-malay.jsonl
2. lom.agc.gov.my PDF files, **this test set is about RAG**, https://huggingface.co/datasets/mesolitica/malaysian-ultrachat/resolve/main/ultrachat-lom-agc.jsonl
3. b.cari.com.my, **this test set is about semantic similarity**, https://huggingface.co/datasets/mesolitica/embedding-pair-mining/resolve/main/b.cari.com.my-dataset-bge-test.jsonl
4. c.cari.com.my, **this test set is about semantic similarity**, https://huggingface.co/datasets/mesolitica/embedding-pair-mining/resolve/main/c.cari.com.my-dataset-bge-test.jsonl
5. Malay news, **this test set is about semantic similarity**, https://huggingface.co/datasets/mesolitica/embedding-pair-mining/resolve/main/malay-news-dataset-bge-test.jsonl
6. Twitter, **this test set is about semantic similarity**, https://huggingface.co/datasets/mesolitica/embedding-pair-mining/resolve/main/twitter-dataset-bge-test.jsonl
"""

close_source = [
    {
        'model': 'OpenAI ADA-002',
        'Crossref Melayu top-1': 0.3155939351340496,
        'Crossref Melayu top-3': 0.5120996083944171,
        'Crossref Melayu top-5': 0.5878100210864544,
        'Crossref Melayu top-10': 0.6721558389396526,
        'lom.agc.gov.my top-1': 0.19168533731640527,
        'lom.agc.gov.my top-3': 0.2827981080408265,
        'lom.agc.gov.my top-5': 0.322504356484939,
        'lom.agc.gov.my top-10': 0.36855862584017923,
        'b.cari.com.my top-1': 0.31621790857858484,
        'b.cari.com.my top-3': 0.6242955541640576,
        'b.cari.com.my top-5': 0.6944270507201001,
        'b.cari.com.my top-10': 0.7623669380087664,
        'c.cari.com.my top-1': 0.08380430943129637,
        'c.cari.com.my top-3': 0.21388202048746027,
        'c.cari.com.my top-5': 0.27861179795125396,
        'c.cari.com.my top-10': 0.3589720946661957,
        'malay-news top-1': 0.1386895659334196,
        'malay-news top-3': 0.2952593812492648,
        'malay-news top-5': 0.3745441712739678,
        'malay-news top-10': 0.4754734737089754,
        'twitter top-1': 0.22942090082094518,
        'twitter top-3': 0.4919014865764367,
        'twitter top-5': 0.5930774351009541,
        'twitter top-10': 0.7248724206789439,
    }
]

open_source = [
    {
        'model': '[mesolitica/llama2-embedding-600m-8k](https://huggingface.co/mesolitica/llama2-embedding-600m-8k)',
        'Crossref Melayu top-1': 0.09549151521237072,
        'Crossref Melayu top-3': 0.1834521538307059,
        'Crossref Melayu top-5': 0.23375840947886334,
        'Crossref Melayu top-10': 0.3098704689225826,
        'lom.agc.gov.my top-1': 0.05215334826985312,
        'lom.agc.gov.my top-3': 0.09932785660941,
        'lom.agc.gov.my top-5': 0.12969878018421707,
        'lom.agc.gov.my top-10': 0.1797361214836943,
        'b.cari.com.my top-1': 0.3168440826549781,
        'b.cari.com.my top-3': 0.6881653099561679,
        'b.cari.com.my top-5': 0.7789605510331872,
        'b.cari.com.my top-10': 0.8453350031308704,
        'c.cari.com.my top-1': 0.08327446132108796,
        'c.cari.com.my top-3': 0.18730130695867184,
        'c.cari.com.my top-5': 0.23975626986930412,
        'c.cari.com.my top-10': 0.3140233133168492,
        'malay-news top-1': 0.14268909540054112,
        'malay-news top-3': 0.27584990001176335,
        'malay-news top-5': 0.3640748147276791,
        'malay-news top-10': 0.47112104458299026,
        'twitter top-1': 0.23496782782338585,
        'twitter top-3': 0.5200798757488352,
        'twitter top-5': 0.6416685156423342,
        'twitter top-10': 0.785888617705791,
    },
    {
        'model': '[mesolitica/llama2-embedding-1b-8k](https://huggingface.co/mesolitica/llama2-embedding-1b-8k)',
        'Crossref Melayu top-1': 0.06777788934631991,
        'Crossref Melayu top-3': 0.142584596847073,
        'Crossref Melayu top-5': 0.18817150316296816,
        'Crossref Melayu top-10': 0.25715433276433375,
        'lom.agc.gov.my top-1': 0.06870799103808813,
        'lom.agc.gov.my top-3': 0.1343042071197411,
        'lom.agc.gov.my top-5': 0.1717699775952203,
        'lom.agc.gov.my top-10': 0.23089370176748816,
        'b.cari.com.my top-1': 0.32623669380087666,
        'b.cari.com.my top-3': 0.6947401377582968,
        'b.cari.com.my top-5': 0.7902316844082655,
        'b.cari.com.my top-10': 0.8603631809643081,
        'c.cari.com.my top-1': 0.08689509007417874,
        'c.cari.com.my top-3': 0.19966442953020133,
        'c.cari.com.my top-5': 0.26086188625927237,
        'c.cari.com.my top-10': 0.3430766513599435,
        'malay-news top-1': 0.14610045876955652,
        'malay-news top-3': 0.285260557581461,
        'malay-news top-5': 0.3640748147276791,
        'malay-news top-10': 0.4775908716621574,
        'twitter top-1': 0.23053028622143332,
        'twitter top-3': 0.5342800088750832,
        'twitter top-5': 0.6498779676059463,
        'twitter top-10': 0.7925449301087197,
    },
    {
        'model': '[mesolitica/mistral-embedding-191m-8k-contrastive](https://huggingface.co/mesolitica/mistral-embedding-191m-8k-contrastive)',
        'Crossref Melayu top-1': 0.08001654088700506,
        'Crossref Melayu top-3': 0.17378269409697095,
        'Crossref Melayu top-5': 0.232192701333609,
        'Crossref Melayu top-10': 0.32482166856197664,
        'lom.agc.gov.my top-1': 0.041075429424943986,
        'lom.agc.gov.my top-3': 0.09148618371919343,
        'lom.agc.gov.my top-5': 0.12758277321384118,
        'lom.agc.gov.my top-10': 0.18707991038088126,
        'b.cari.com.my top-1': 0.2564182842830307,
        'b.cari.com.my top-3': 0.46900438321853477,
        'b.cari.com.my top-5': 0.5309956167814652,
        'b.cari.com.my top-10': 0.6036318096430808,
        'c.cari.com.my top-1': 0.05298481102084069,
        'c.cari.com.my top-3': 0.10005298481102085,
        'c.cari.com.my top-5': 0.12433768986223949,
        'c.cari.com.my top-10': 0.16990462734016248,
        'malay-news top-1': 0.1102223267850841,
        'malay-news top-3': 0.1672744383013763,
        'malay-news top-5': 0.20091753911304552,
        'malay-news top-10': 0.24973532525585226,
        'twitter top-1': 0.18437985356112713,
        'twitter top-3': 0.3536720656756157,
        'twitter top-5': 0.422897714666075,
        'twitter top-10': 0.5227424007100067,
    },
    {
        'model': '[mesolitica/mistral-embedding-349m-8k-contrastive](https://huggingface.co/mesolitica/mistral-embedding-349m-8k-contrastive)',
        'Crossref Melayu top-1': 0.09045797580895276,
        'Crossref Melayu top-3': 0.18742892587615012,
        'Crossref Melayu top-5': 0.2444949860436266,
        'Crossref Melayu top-10': 0.3398118474103174,
        'lom.agc.gov.my top-1': 0.039581777445855115,
        'lom.agc.gov.my top-3': 0.08849887976101568,
        'lom.agc.gov.my top-5': 0.12335075927308937,
        'lom.agc.gov.my top-10': 0.18558625840179238,
        'b.cari.com.my top-1': 0.24953036944270507,
        'b.cari.com.my top-3': 0.4730745147150908,
        'b.cari.com.my top-5': 0.5341264871634315,
        'b.cari.com.my top-10': 0.5970569818409518,
        'c.cari.com.my top-1': 0.057400211939244085,
        'c.cari.com.my top-3': 0.10588131402331331,
        'c.cari.com.my top-5': 0.13149063935005298,
        'c.cari.com.my top-10': 0.17582126457082303,
        'malay-news top-1': 0.11375132337372074,
        'malay-news top-3': 0.17362663216092225,
        'malay-news top-5': 0.20574050111751557,
        'malay-news top-10': 0.26149864721797433,
        'twitter top-1': 0.18105169735966276,
        'twitter top-3': 0.3547814510761038,
        'twitter top-5': 0.42666962502773464,
        'twitter top-10': 0.5229642777901042,
    },
    {
        'model': '[mesolitica/llama2-embedding-600m-8k-contrastive](https://huggingface.co/mesolitica/llama2-embedding-600m-8k-contrastive)',
        'Crossref Melayu top-1': 0.1260208828698439,
        'Crossref Melayu top-3': 0.2649643337123953,
        'Crossref Melayu top-5': 0.3535614597332782,
        'Crossref Melayu top-10': 0.4751369792205107,
        'lom.agc.gov.my top-1': 0.0660941000746826,
        'lom.agc.gov.my top-3': 0.13754045307443366,
        'lom.agc.gov.my top-5': 0.18869803335822755,
        'lom.agc.gov.my top-10': 0.2745830221558377,
        'b.cari.com.my top-1': 0.254539762053851,
        'b.cari.com.my top-3': 0.4815278647463995,
        'b.cari.com.my top-5': 0.5497808390732624,
        'b.cari.com.my top-10': 0.628052598622417,
        'c.cari.com.my top-1': 0.05183680678205581,
        'c.cari.com.my top-3': 0.09766866831508302,
        'c.cari.com.my top-5': 0.12398445778876722,
        'c.cari.com.my top-10': 0.16566584245849522,
        'malay-news top-1': 0.11657452064463004,
        'malay-news top-3': 0.18986001646865075,
        'malay-news top-5': 0.22338548406069875,
        'malay-news top-10': 0.2812610281143395,
        'twitter top-1': 0.1961393388063013,
        'twitter top-3': 0.393831817173286,
        'twitter top-5': 0.48080763257155534,
        'twitter top-10': 0.5977368537830042,
    },
    {
        'model': '[mesolitica/llama2-embedding-1b-8k-contrastive](https://huggingface.co/mesolitica/llama2-embedding-1b-8k-contrastive)',
        'Crossref Melayu top-1': 0.21958027499224647,
        'Crossref Melayu top-3': 0.42809883179985525,
        'Crossref Melayu top-5': 0.5379923498397602,
        'Crossref Melayu top-10': 0.6708363485991936,
        'lom.agc.gov.my top-1': 0.11899427433408016,
        'lom.agc.gov.my top-3': 0.239731142643764,
        'lom.agc.gov.my top-5': 0.3094349016679114,
        'lom.agc.gov.my top-10': 0.4134926562111028,
        'b.cari.com.my top-1': 0.29805886036318097,
        'b.cari.com.my top-3': 0.5795241077019412,
        'b.cari.com.my top-5': 0.6581089542892924,
        'b.cari.com.my top-10': 0.7366938008766437,
        'c.cari.com.my top-1': 0.06720240197809961,
        'c.cari.com.my top-3': 0.13678912045213706,
        'c.cari.com.my top-5': 0.1750264924055104,
        'c.cari.com.my top-10': 0.22818791946308725,
        'malay-news top-1': 0.12892600870485826,
        'malay-news top-3': 0.20785789907069757,
        'malay-news top-5': 0.24808846018115516,
        'malay-news top-10': 0.32466768615457003,
        'twitter top-1': 0.2167739072553805,
        'twitter top-3': 0.4453072997559352,
        'twitter top-5': 0.5387175504770357,
        'twitter top-10': 0.6685156423341468,
    },
    {
        'model': '[mesolitica/llama2-embedding-2b-8k-contrastive](https://huggingface.co/mesolitica/llama2-embedding-2b-8k-contrastive)',
        'Crossref Melayu top-1': 0.2671353251318102,
        'Crossref Melayu top-3': 0.49116096350666805,
        'Crossref Melayu top-5': 0.6091181639615424,
        'Crossref Melayu top-10': 0.739791171301561,
        'lom.agc.gov.my top-1': 0.1549663928304705,
        'lom.agc.gov.my top-3': 0.29113766492407267,
        'lom.agc.gov.my top-5': 0.3725416977844162,
        'lom.agc.gov.my top-10': 0.4886731391585761,
        'b.cari.com.my top-1': 0.30713838447088293,
        'b.cari.com.my top-3': 0.6361928616155291,
        'b.cari.com.my top-5': 0.7213525360050094,
        'b.cari.com.my top-10': 0.8015028177833438,
        'c.cari.com.my top-1': 0.07444365948428117,
        'c.cari.com.my top-3': 0.15745319675026492,
        'c.cari.com.my top-5': 0.19931119745672907,
        'c.cari.com.my top-10': 0.26960438007771104,
        'malay-news top-1': 0.13633690154099518,
        'malay-news top-3': 0.23056111045759323,
        'malay-news top-5': 0.28726032231502174,
        'malay-news top-10': 0.3717209740030585,
        'twitter top-1': 0.22498335921899268,
        'twitter top-3': 0.48679831373419125,
        'twitter top-5': 0.5850898602174396,
        'twitter top-10': 0.7333037497226537,
    },
    {
        'model': '[BAAI/bge-m3](https://huggingface.co/BAAI/bge-m3)',
        'Crossref Melayu top-1': 0.41331541403907784,
        'Crossref Melayu top-3': 0.669802543161377,
        'Crossref Melayu top-5': 0.7698749095420242,
        'Crossref Melayu top-10': 0.8632275405768635,
        'lom.agc.gov.my top-1': 0.35001244709982576,
        'lom.agc.gov.my top-3': 0.5311177495643515,
        'lom.agc.gov.my top-5': 0.6011949215832711,
        'lom.agc.gov.my top-10': 0.6928055763007219,
        'b.cari.com.my top-1': 0.3306199123356293,
        'b.cari.com.my top-3': 0.7025673137132122,
        'b.cari.com.my top-5': 0.7930494677520351,
        'b.cari.com.my top-10': 0.8647463994990607,
        'c.cari.com.my top-1': 0.08795478629459555,
        'c.cari.com.my top-3': 0.23754856941010244,
        'c.cari.com.my top-5': 0.33574708583539387,
        'c.cari.com.my top-10': 0.4558459908159661,
        'malay-news top-1': 0.1459828255499353,
        'malay-news top-3': 0.3185507587342666,
        'malay-news top-5': 0.4232443241971533,
        'malay-news top-10': 0.5473473708975415,
        'twitter top-1': 0.22919902374084758,
        'twitter top-3': 0.5225205236299091,
        'twitter top-5': 0.6443310406035057,
        'twitter top-10': 0.7836698469048148,
    }
]

data = pd.DataFrame(close_source + open_source)

demo = gr.Blocks(css=custom_css)
with demo:
    gr.HTML(TITLE)
    gr.Markdown(INTRODUCTION_TEXT, elem_classes="markdown-text")
    gr.DataFrame(data, datatype = 'markdown')

demo.launch()