Add retrieval and reranking leaderboard modules, update requirements and README

.gitignore

@@ -0,0 +1,177 @@
+# Created by https://www.toptal.com/developers/gitignore/api/python
+# Edit at https://www.toptal.com/developers/gitignore?templates=python
+
+### Python ###
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py,cover
+.hypothesis/
+.pytest_cache/
+cover/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+.pybuilder/
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+#   For a library or package, you might want to ignore these files since the code is
+#   intended to run in multiple environments; otherwise, check them in:
+# .python-version
+
+# pipenv
+#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+#   However, in case of collaboration, if having platform-specific dependencies or dependencies
+#   having no cross-platform support, pipenv may install dependencies that don't work, or not
+#   install all needed dependencies.
+#Pipfile.lock
+
+# poetry
+#   Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
+#   This is especially recommended for binary packages to ensure reproducibility, and is more
+#   commonly ignored for libraries.
+#   https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
+#poetry.lock
+
+# pdm
+#   Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
+#pdm.lock
+#   pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it
+#   in version control.
+#   https://pdm.fming.dev/#use-with-ide
+.pdm.toml
+
+# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
+__pypackages__/
+
+# Celery stuff
+celerybeat-schedule
+celerybeat.pid
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
+# pytype static type analyzer
+.pytype/
+
+# Cython debug symbols
+cython_debug/
+
+# PyCharm
+#  JetBrains specific template is maintained in a separate JetBrains.gitignore that can
+#  be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
+#  and can be added to the global gitignore or merged into this file.  For a more nuclear
+#  option (not recommended) you can uncomment the following to ignore the entire idea folder.
+#.idea/
+
+### Python Patch ###
+# Poetry local configuration file - https://python-poetry.org/docs/configuration/#local-configuration
+poetry.toml
+
+# ruff
+.ruff_cache/
+
+# LSP config files
+pyrightconfig.json
+
+# .env file
+.env

README.md

@@ -4,7 +4,7 @@ emoji: 📊
 colorFrom: green
 colorTo: indigo
 sdk: gradio
-sdk_version: 5.14.0
+sdk_version: 5.24.0
 app_file: app.py
 pinned: true
 short_description: The only leaderboard you will require for your RAG needs 🏆

app.py

@@ -1,6 +1,9 @@
 import gradio as gr
-from utils import submit_gradio_module, load_retrieval_results, load_reranking_results
-from fuzzywuzzy import fuzz
+from retrieval_leaderboard import create_retrieval_tab
+from reranking_leaderboard import create_reranking_tab
+from llm_in_context_leaderboard import create_llm_in_context_tab
+from dotenv import load_dotenv
+load_dotenv()
 
 HEADER = """<div style="text-align: center; margin-bottom: 20px;">
     <h1>The Arabic RAG Leaderboard</h1>
@@ -13,68 +16,6 @@ This leaderboard presents the first comprehensive benchmark for Arabic RAG syste
 For technical details, check our blog post <a href="https://huggingface.co/blog/Navid-AI/arabic-rag-leaderboard">here</a>.
 """
 
-RETRIEVAL_ABOUT_SECTION = """
-## About Retrieval Evaluation
-
-The retrieval evaluation assesses a model's ability to find and retrieve relevant information from a large corpus of Arabic text. Models are evaluated on:
-
-### Web Search Dataset Metrics
-- **MRR (Mean Reciprocal Rank)**: Measures the ranking quality by focusing on the position of the first relevant result
-- **nDCG (Normalized Discounted Cumulative Gain)**: Evaluates the ranking quality considering all relevant results
-- **Recall@5**: Measures the proportion of relevant documents found in the top 5 results
-- **Overall Score**: Combined score calculated as the average of MRR, nDCG, and Recall@5
-
-### Model Requirements
-- Must support Arabic text embeddings
-- Should handle queries of at least 512 tokens 
-- Must work with `sentence-transformers` library
-
-### Evaluation Process
-1. Models process Arabic web search queries
-2. Retrieved documents are evaluated using:
-   - MRR for first relevant result positioning
-   - nDCG for overall ranking quality
-   - Recall@5 for top results accuracy
-3. Metrics are averaged to calculate the overall score
-4. Models are ranked based on their overall performance
-
-### How to Prepare Your Model
-- Ensure your model is publicly available on HuggingFace Hub (We don't support private model evaluations yet)
-- Model should output fixed-dimension embeddings for text
-- Support batch processing for efficient evaluation (this is default if you use `sentence-transformers`)
-"""
-
-RERANKER_ABOUT_SECTION = """
-## About Reranking Evaluation
-
-The reranking evaluation assesses a model's ability to improve search quality by reordering initially retrieved results. Models are evaluated across multiple unseen Arabic datasets to ensure robust performance.
-
-### Evaluation Metrics
-- **MRR@10 (Mean Reciprocal Rank at 10)**: Measures the ranking quality focusing on the first relevant result in top-10
-- **NDCG@10 (Normalized DCG at 10)**: Evaluates the ranking quality of all relevant results in top-10
-- **MAP (Mean Average Precision)**: Measures the overall precision across all relevant documents
-
-All metrics are averaged across multiple evaluation datasets to provide a comprehensive assessment of model performance.
-
-### Model Requirements
-- Must accept query-document pairs as input
-- Should output relevance scores for reranking (has cross-attention or similar mechanism for query-document matching)
-- Support for Arabic text processing
-
-### Evaluation Process
-1. Models are tested on multiple unseen Arabic datasets
-2. For each dataset:
-   - Initial candidate documents are provided
-   - Model reranks the candidates
-   - MRR@10, NDCG@10, and MAP are calculated
-3. Final scores are averaged across all datasets
-4. Models are ranked based on overall performance
-
-### How to Prepare Your Model
-- Model should be public on HuggingFace Hub (private models are not supported yet)
-- Make sure it works coherently with `sentence-transformers` library
-"""
-
 CITATION_BUTTON_LABEL = """
 Copy the following snippet to cite these results
 """
@@ -89,162 +30,20 @@ CITATION_BUTTON_TEXT = """
 }
 """
 
-retrieval_df = None
-reranking_df = None
-
-def search_leaderboard(df, model_name, columns_to_show, threshold=95):
-    if not model_name.strip():
-        return df.loc[:, columns_to_show]
-    search_name = model_name.lower()  # compute once for efficiency
-    def calculate_similarity(row):
-        return fuzz.partial_ratio(search_name, row["Model"].lower())
-    filtered_df = df.copy()
-    filtered_df["similarity"] = filtered_df.apply(calculate_similarity, axis=1)
-    filtered_df = filtered_df[filtered_df["similarity"] >= threshold].sort_values('similarity', ascending=False)
-    filtered_df = filtered_df.drop('similarity', axis=1).loc[:, columns_to_show]
-    return filtered_df
-
-def retrieval_search_leaderboard(model_name, columns_to_show):
-    return search_leaderboard(retrieval_df, model_name, columns_to_show)
-
-def reranking_search_leaderboard(model_name, columns_to_show):
-    return search_leaderboard(reranking_df, model_name, columns_to_show)
-
-def update_retrieval_columns_to_show(columns_to_show):
-    global retrieval_df
-    dummy_df = retrieval_df.loc[:, [col for col in retrieval_df.columns if col in columns_to_show]]
-    columns_widths = []
-    for col in dummy_df.columns:
-        if col == "Rank":
-            columns_widths.append(80)
-        elif col == "Model":
-            columns_widths.append(400)
-        else:
-            columns_widths.append(150)
-    return gr.update(value=dummy_df, column_widths=columns_widths)
-
-def update_reranker_columns_to_show(columns_to_show):
-    global reranking_df
-    dummy_df = reranking_df.loc[:, [col for col in reranking_df.columns if col in columns_to_show]]
-    columns_widths = []
-    for col in dummy_df.columns:
-        if col == "Rank":
-            columns_widths.append(80)
-        elif col == "Model":
-            columns_widths.append(400)
-        else:
-            columns_widths.append(150)
-    return gr.update(value=dummy_df, column_widths=columns_widths)
-
 def main():
-    global retrieval_df, reranking_df
-
-    # Prepare retrieval dataframe
-    retrieval_df = load_retrieval_results(True, "Web Search Dataset (Overall Score)", ["Revision", "Precision", "Task"])
-    retrieval_df.insert(0, "Rank", range(1, 1 + len(retrieval_df)))
-    retrieval_df = retrieval_df[['Rank', 'Model', 'Web Search Dataset (Overall Score)', 'Model Size (MB)', 'Embedding Dimension', 'Max Tokens', 'Num Likes', 'Downloads Last Month', 'Web Search Dataset (MRR)', 'Web Search Dataset (nDCG@k=None)', 'Web Search Dataset (Recall@5)', 'License']]
-    retrieval_columns_to_show = ["Rank", "Model", "Web Search Dataset (Overall Score)", "Model Size (MB)", "Embedding Dimension", "Max Tokens", "Num Likes"]
-    retrieval_columns_widths = [80, 400, 150, 150, 150, 150, 150]
-    retrieval_cols = retrieval_df.columns.tolist()  # cache columns
-
-    # Prepare reranking dataframe
-    reranking_df = load_reranking_results(True, sort_col="Overall Score", drop_cols=["Revision", "Precision", "Task"])
-    reranking_df.insert(0, "Rank", range(1, 1 + len(reranking_df)))
-    reranking_df.rename(columns={"nDCG": "nDCG@10", "MRR": "MRR@10"}, inplace=True)
-    reranking_columns_to_show = ["Rank", "Model", "Overall Score", "Model Parameters (in Millions)", "Embedding Dimensions", "Downloads Last Month", "MRR@10", "nDCG@10", "MAP"]
-    reranking_columns_widths = [80, 400, 150, 150, 150, 150, 150, 150, 150]
-    reranking_cols = reranking_df.columns.tolist()  # cache columns
-
     with gr.Blocks() as demo:
         gr.HTML(HEADER)
         
         with gr.Tabs():
             with gr.Tab("🕵️‍♂️ Retrieval"):
-                with gr.Tabs():
-                    with gr.Tab("👑 Leaderboard"):
-                        with gr.Row():
-                            search_box_retrieval = gr.Textbox(
-                                placeholder="Search for models...", 
-                                label="Search", 
-                                scale=5
-                            )
-                            retrieval_columns_to_show_input = gr.CheckboxGroup(
-                                label="Columns to Show",
-                                choices=retrieval_cols,  # use cached list
-                                value=retrieval_columns_to_show,
-                                scale=4
-                            )
-
-                        retrieval_leaderboard = gr.Dataframe(
-                            value=retrieval_df.loc[:, retrieval_columns_to_show],
-                            datatype="markdown",
-                            wrap=False,
-                            show_fullscreen_button=True,
-                            interactive=False,
-                            column_widths=retrieval_columns_widths
-                        )
-                        
-                        # Submit the search box and the leaderboard
-                        search_box_retrieval.input(
-                            retrieval_search_leaderboard,
-                            inputs=[search_box_retrieval, retrieval_columns_to_show_input],
-                            outputs=retrieval_leaderboard
-                        )
-                        retrieval_columns_to_show_input.select(
-                            update_retrieval_columns_to_show,
-                            inputs=retrieval_columns_to_show_input,
-                            outputs=retrieval_leaderboard
-                        )
-                        
-                    with gr.Tab("🏵️ Submit Retriever"):
-                        submit_gradio_module("Retriever")
-
-                    with gr.Tab("ℹ️ About"):
-                        gr.Markdown(RETRIEVAL_ABOUT_SECTION)
+                create_retrieval_tab()
 
             with gr.Tab("📊 Reranking"):
-                with gr.Tabs():
-                    with gr.Tab("👑 Leaderboard"):
-                        with gr.Row():
-                            search_box_reranker = gr.Textbox(
-                                placeholder="Search for models...", 
-                                label="Search", 
-                                scale=5
-                            )
-                            reranking_columns_to_show_input = gr.CheckboxGroup(
-                                label="Columns to Show",
-                                choices=reranking_cols,  # use cached list
-                                value=reranking_columns_to_show,
-                                scale=4
-                            )
-                        
-                        reranker_leaderboard = gr.Dataframe(
-                            value=reranking_df[reranking_columns_to_show],
-                            datatype="markdown",
-                            wrap=False,
-                            show_fullscreen_button=True,
-                            interactive=False,
-                            column_widths=reranking_columns_widths
-                        )
-
-                        # Submit the search box and the leaderboard
-                        search_box_reranker.input(
-                            reranking_search_leaderboard,
-                            inputs=[search_box_reranker, reranking_columns_to_show_input],
-                            outputs=reranker_leaderboard
-                        )
-                        reranking_columns_to_show_input.select(
-                            update_reranker_columns_to_show,
-                            inputs=reranking_columns_to_show_input,
-                            outputs=reranker_leaderboard
-                        )
-
-                    with gr.Tab("🏵️ Submit Reranker"):
-                        submit_gradio_module("Reranker")
-
-                    with gr.Tab("ℹ️ About"):
-                        gr.Markdown(RERANKER_ABOUT_SECTION)
+                create_reranking_tab()
                     
+            # with gr.Tab("📊 LLM in Context"):
+            #     create_llm_in_context_tab()
+
             with gr.Row():
                 with gr.Accordion("📙 Citation", open=False):
                     gr.Textbox(

leaderboard_tab.py

@@ -0,0 +1,122 @@
+import gradio as gr
+import pandas as pd
+from fuzzywuzzy import fuzz
+from utils import submit_gradio_module
+
+def search_leaderboard(df, model_name, columns_to_show, threshold=95):
+    """
+    Search the leaderboard for models matching the search term using fuzzy matching.
+    
+    Args:
+        df: The dataframe containing all leaderboard data
+        model_name: The search term to find models
+        columns_to_show: List of columns to include in the result
+        threshold: Minimum similarity threshold (default: 95)
+    
+    Returns:
+        Filtered dataframe with only matching models and selected columns
+    """
+    if not model_name.strip():
+        return df.loc[:, columns_to_show]
+    search_name = model_name.lower()  # compute once for efficiency
+    def calculate_similarity(row):
+        return fuzz.partial_ratio(search_name, row["Model"].lower())
+    filtered_df = df.copy()
+    filtered_df["similarity"] = filtered_df.apply(calculate_similarity, axis=1)
+    filtered_df = filtered_df[filtered_df["similarity"] >= threshold].sort_values('similarity', ascending=False)
+    filtered_df = filtered_df.drop('similarity', axis=1).loc[:, columns_to_show]
+    return filtered_df
+
+def update_columns_to_show(df, columns_to_show):
+    """
+    Update the displayed columns in the dataframe.
+    
+    Args:
+        df: The dataframe to update
+        columns_to_show: List of columns to include
+    
+    Returns:
+        gradio.update object with the updated dataframe
+    """
+    dummy_df = df.loc[:, [col for col in df.columns if col in columns_to_show]]
+    columns_widths = []
+    for col in dummy_df.columns:
+        if col == "Rank":
+            columns_widths.append(80)
+        elif col == "Model":
+            columns_widths.append(400)
+        else:
+            columns_widths.append(150)
+    return gr.update(value=dummy_df, column_widths=columns_widths)
+
+def create_leaderboard_tab(df, initial_columns_to_show, search_function, update_function, about_section, task_type):
+    """
+    Create a complete leaderboard tab with search, column selection, and data display.
+    
+    Args:
+        df: The dataframe containing the leaderboard data
+        initial_columns_to_show: Initial list of columns to display
+        search_function: Function to handle searching
+        update_function: Function to handle column updates
+        about_section: Markdown text for the About tab
+        task_type: Type of the task ("Retriever" or "Reranker")
+    
+    Returns:
+        A gradio Tabs component with the complete leaderboard interface
+    """
+    columns_widths = [80 if col == "Rank" else 400 if col == "Model" else 150 for col in initial_columns_to_show]
+    
+    with gr.Tabs() as tabs:
+        with gr.Tab("👑 Leaderboard"):
+            with gr.Column():
+                with gr.Row(equal_height=True):
+                    search_box = gr.Textbox(
+                        placeholder="Search for models...", 
+                        label="Search (You can also press Enter to search)", 
+                        scale=5
+                    )
+                    search_button = gr.Button(
+                        value="Search",
+                        variant="primary",
+                        scale=1
+                    )
+                columns_to_show_input = gr.CheckboxGroup(
+                    label="Columns to Show",
+                    choices=df.columns.tolist(),
+                    value=initial_columns_to_show,
+                    scale=4
+                )
+
+            leaderboard = gr.Dataframe(
+                value=df.loc[:, initial_columns_to_show],
+                datatype="markdown",
+                wrap=True,
+                show_fullscreen_button=True,
+                interactive=False,
+                column_widths=columns_widths
+            )
+            
+            # Connect events
+            search_box.submit(
+                search_function,
+                inputs=[search_box, columns_to_show_input],
+                outputs=leaderboard
+            )
+            columns_to_show_input.select(
+                update_function,
+                inputs=columns_to_show_input,
+                outputs=leaderboard
+            )
+            search_button.click(
+                search_function,
+                inputs=[search_box, columns_to_show_input],
+                outputs=leaderboard
+            )
+            
+        with gr.Tab("🏵️ Submit"):
+            submit_gradio_module(task_type)
+
+        with gr.Tab("ℹ️ About"):
+            gr.Markdown(about_section)
+    
+    return tabs

llm_in_context_leaderboard.py

@@ -0,0 +1,152 @@
+from pathlib import Path
+from utils import load_json_results
+import gradio as gr
+from leaderboard_tab import search_leaderboard, update_columns_to_show, create_leaderboard_tab
+
+# Constants
+LLM_IN_CONTEXT_ABOUT_SECTION = """"""
+
+# Global variables
+llm_in_context_df = None
+
+def load_reranking_leaderboard():
+    """Load and prepare the reranking leaderboard data"""
+    global llm_in_context_df
+    
+    dataframe_path = Path(__file__).parent / "results" / "llm_in_context_results.json"
+    
+    # Prepare dataframe
+    llm_in_context_df = load_json_results(
+        dataframe_path,
+        prepare_for_display=True,
+        sort_col="Overall Score",
+        drop_cols=["Revision", "Precision", "Task"]
+    )
+    llm_in_context_df.insert(0, "Rank", range(1, 1 + len(llm_in_context_df)))
+    llm_in_context_df.rename(columns={"nDCG": "nDCG@10", "MRR": "MRR@10"}, inplace=True)
+    
+    return llm_in_context_df
+
+def reranking_search_leaderboard(model_name, columns_to_show):
+    """Search function for reranking leaderboard"""
+    return search_leaderboard(llm_in_context_df, model_name, columns_to_show)
+
+def update_reranker_columns_to_show(columns_to_show):
+    """Update displayed columns for reranking leaderboard"""
+    return update_columns_to_show(llm_in_context_df, columns_to_show)
+
+def create_llm_in_context_tab():
+    """Create the complete reranking leaderboard tab"""
+    global llm_in_context_df
+    
+    # Load data if not already loaded
+    if (llm_in_context_df is None):
+        llm_in_context_df = load_reranking_leaderboard()
+    
+    # Define default columns to show
+    default_columns = ["Rank", "Model", "Overall Score", "Model Parameters (in Millions)", 
+                      "Embedding Dimensions", "Downloads Last Month", "MRR@10", "nDCG@10", "MAP"]
+    
+    columns_widths = [80 if col == "Rank" else 400 if col == "Model" else 150 for col in initial_columns_to_show]
+    
+    with gr.Tabs() as tabs:
+        with gr.Tab("👑 Context Dependant Leaderboard"):
+            with gr.Column():
+                with gr.Row(equal_height=True):
+                    search_box = gr.Textbox(
+                        placeholder="Search for models...", 
+                        label="Search (You can also press Enter to search)", 
+                        scale=5
+                    )
+                    search_button = gr.Button(
+                        value="Search",
+                        variant="primary",
+                        scale=1
+                    )
+                columns_to_show_input = gr.CheckboxGroup(
+                    label="Columns to Show",
+                    choices=llm_in_context_df.columns.tolist(),
+                    value=initial_columns_to_show,
+                    scale=4
+                )
+
+            leaderboard = gr.Dataframe(
+                value=llm_in_context_df.loc[:, initial_columns_to_show],
+                datatype="markdown",
+                wrap=False,
+                show_fullscreen_button=True,
+                interactive=False,
+                column_widths=columns_widths
+            )
+            
+            # Connect events
+            search_box.submit(
+                search_function,
+                inputs=[search_box, columns_to_show_input],
+                outputs=leaderboard
+            )
+            columns_to_show_input.select(
+                update_function,
+                inputs=columns_to_show_input,
+                outputs=leaderboard
+            )
+            search_button.click(
+                search_function,
+                inputs=[search_box, columns_to_show_input],
+                outputs=leaderboard
+            )
+        
+        with gr.Tab("💎 Context About Leaderboard"):
+            with gr.Column():
+                with gr.Row(equal_height=True):
+                    search_box = gr.Textbox(
+                        placeholder="Search for models...", 
+                        label="Search (You can also press Enter to search)", 
+                        scale=5
+                    )
+                    search_button = gr.Button(
+                        value="Search",
+                        variant="primary",
+                        scale=1
+                    )
+                columns_to_show_input = gr.CheckboxGroup(
+                    label="Columns to Show",
+                    choices=llm_in_context_df.columns.tolist(),
+                    value=initial_columns_to_show,
+                    scale=4
+                )
+
+            leaderboard = gr.Dataframe(
+                value=llm_in_context_df.loc[:, initial_columns_to_show],
+                datatype="markdown",
+                wrap=False,
+                show_fullscreen_button=True,
+                interactive=False,
+                column_widths=columns_widths
+            )
+            
+            # Connect events
+            search_box.submit(
+                search_function,
+                inputs=[search_box, columns_to_show_input],
+                outputs=leaderboard
+            )
+            columns_to_show_input.select(
+                update_function,
+                inputs=columns_to_show_input,
+                outputs=leaderboard
+            )
+            search_button.click(
+                search_function,
+                inputs=[search_box, columns_to_show_input],
+                outputs=leaderboard
+            )
+
+        with gr.Tab("🏵️ Submit"):
+            submit_gradio_module(task_type)
+
+        with gr.Tab("ℹ️ About"):
+            gr.Markdown(about_section)
+    
+    return tabs
+

requirements.txt

@@ -1,2 +1,3 @@
 fuzzywuzzy
-Levenshtein
+Levenshtein
+python-dotenv

reranking_leaderboard.py

@@ -0,0 +1,87 @@
+from pathlib import Path
+from utils import load_json_results
+from leaderboard_tab import search_leaderboard, update_columns_to_show, create_leaderboard_tab
+
+# Constants
+RERANKER_ABOUT_SECTION = """
+## About Reranking Evaluation
+
+The reranking evaluation assesses a model's ability to improve search quality by reordering initially retrieved results. Models are evaluated across multiple unseen Arabic datasets to ensure robust performance.
+
+### Evaluation Metrics
+- **MRR@10 (Mean Reciprocal Rank at 10)**: Measures the ranking quality focusing on the first relevant result in top-10
+- **NDCG@10 (Normalized DCG at 10)**: Evaluates the ranking quality of all relevant results in top-10
+- **MAP (Mean Average Precision)**: Measures the overall precision across all relevant documents
+
+All metrics are averaged across multiple evaluation datasets to provide a comprehensive assessment of model performance.
+
+### Model Requirements
+- Must accept query-document pairs as input
+- Should output relevance scores for reranking (has cross-attention or similar mechanism for query-document matching)
+- Support for Arabic text processing
+
+### Evaluation Process
+1. Models are tested on multiple unseen Arabic datasets
+2. For each dataset:
+   - Initial candidate documents are provided
+   - Model reranks the candidates
+   - MRR@10, NDCG@10, and MAP are calculated
+3. Final scores are averaged across all datasets
+4. Models are ranked based on overall performance
+
+### How to Prepare Your Model
+- Model should be public on HuggingFace Hub (private models are not supported yet)
+- Make sure it works coherently with `sentence-transformers` library
+"""
+
+# Global variables
+reranking_df = None
+
+def load_reranking_results(prepare_for_display=False, sort_col=None, drop_cols=None):
+    dataframe_path = Path(__file__).parent / "results" / "reranking_results.json"
+    return load_json_results(
+        dataframe_path,
+        prepare_for_display=prepare_for_display,
+        sort_col=sort_col,
+        drop_cols=drop_cols
+    )
+
+def load_reranking_leaderboard():
+    """Load and prepare the reranking leaderboard data"""
+    global reranking_df
+    
+    # Prepare reranking dataframe
+    reranking_df = load_reranking_results(True, sort_col="Average Score", drop_cols=["Revision", "Precision", "Task"])
+    reranking_df.insert(0, "Rank", range(1, 1 + len(reranking_df)))
+    
+    return reranking_df
+
+def reranking_search_leaderboard(model_name, columns_to_show):
+    """Search function for reranking leaderboard"""
+    return search_leaderboard(reranking_df, model_name, columns_to_show)
+
+def update_reranker_columns_to_show(columns_to_show):
+    """Update displayed columns for reranking leaderboard"""
+    return update_columns_to_show(reranking_df, columns_to_show)
+
+def create_reranking_tab():
+    """Create the complete reranking leaderboard tab"""
+    global reranking_df
+    
+    # Load data if not already loaded
+    if (reranking_df is None):
+        reranking_df = load_reranking_leaderboard()
+    
+    # Define default columns to show
+    default_columns = ["Rank", "Model", "Average Score", "Model Size (MB)", "Context Length",
+                      "Embedding Dimension", "Namaa Global Knowledge", "Navid General Knowledge"]
+    
+    # Create and return the tab
+    return create_leaderboard_tab(
+        df=reranking_df,
+        initial_columns_to_show=default_columns,
+        search_function=reranking_search_leaderboard,
+        update_function=update_reranker_columns_to_show,
+        about_section=RERANKER_ABOUT_SECTION,
+        task_type="Reranker"
+    )

results/reranking_results.json

@@ -1,242 +1,506 @@
 [
+    {
+        "Model": "Alibaba-NLP/gte-multilingual-reranker-base",
+        "Revision": "main",
+        "Precision": "f16",
+        "Task": "Reranker",
+        "Average Score": 85.03,
+        "Context Length": 8192,
+        "Embedding Dimension": 768,
+        "Model Size (MB)": 582.44,
+        "Number of Parameters (Billions)": 0.305,
+        "Namaa Global Knowledge": 76.76,
+        "Namaa Tourism": 77.1,
+        "Namaa Media": 78.88,
+        "Namaa Legal": 85.87,
+        "Navid General Knowledge": 86.05,
+        "Navid Web Search": 99.36,
+        "Navid Islamic Studies": 91.18,
+        "Downloads Last Month": 215347,
+        "Num Likes": 114,
+        "License": "apache-2.0"
+    },
     {
         "Model": "BAAI/bge-reranker-v2-m3",
-        "Overall Score": 85.4,
-        "Embedding Dimensions": 1024,
-        "Model Parameters (in Millions)": 568.0,
-        "Downloads Last Month": 966662,
-        "MRR": 79.41,
-        "nDCG": 95.1,
-        "MAP": 81.69,
-        "Num Likes": 491,
-        "License": "apache-2.0",
-        "Precision": "F32",
+        "Revision": "main",
+        "Precision": "f32",
         "Task": "Reranker",
-        "Revision": "main"
+        "Average Score": 87.44,
+        "Context Length": 8192,
+        "Embedding Dimension": 1024,
+        "Model Size (MB)": 2165.81,
+        "Number of Parameters (Billions)": 0.568,
+        "Namaa Global Knowledge": 81.27,
+        "Namaa Tourism": 80.96,
+        "Namaa Media": 81.33,
+        "Namaa Legal": 88.58,
+        "Navid General Knowledge": 87.05,
+        "Navid Web Search": 99.3,
+        "Navid Islamic Studies": 93.59,
+        "Downloads Last Month": 1480691,
+        "Num Likes": 597,
+        "License": "apache-2.0"
     },
     {
-        "Model": "NAMAA-Space/Namaa-ARA-Reranker-V1",
-        "Overall Score": 84.1,
-        "Embedding Dimensions": 1024,
-        "Model Parameters (in Millions)": 568.0,
-        "Downloads Last Month": 121,
-        "MRR": 76.48,
-        "nDCG": 93.14,
-        "MAP": 82.67,
-        "Num Likes": 4,
-        "License": "apache-2.0",
-        "Precision": "F32",
+        "Model": "Lajavaness/bilingual-embedding-large",
+        "Revision": "main",
+        "Precision": "f32",
         "Task": "Reranker",
-        "Revision": "main"
+        "Average Score": 47.37,
+        "Context Length": 512,
+        "Embedding Dimension": 1024,
+        "Model Size (MB)": 2135.81,
+        "Number of Parameters (Billions)": 0.56,
+        "Namaa Global Knowledge": 46.48,
+        "Namaa Tourism": 46.61,
+        "Namaa Media": 46.01,
+        "Namaa Legal": 49.64,
+        "Navid General Knowledge": 47.87,
+        "Navid Web Search": 48.37,
+        "Navid Islamic Studies": 46.6,
+        "Downloads Last Month": 24311,
+        "Num Likes": 21,
+        "License": "apache-2.0"
     },
     {
         "Model": "NAMAA-Space/GATE-Reranker-V1",
-        "Overall Score": 76.81,
-        "Embedding Dimensions": 768,
-        "Model Parameters (in Millions)": 135.0,
-        "Downloads Last Month": 753,
-        "MRR": 62.61,
-        "nDCG": 89.7,
-        "MAP": 78.11,
+        "Revision": "main",
+        "Precision": "f32",
+        "Task": "Reranker",
+        "Average Score": 83.96,
+        "Context Length": 512,
+        "Embedding Dimension": 768,
+        "Model Size (MB)": 515.72,
+        "Number of Parameters (Billions)": 0.135,
+        "Namaa Global Knowledge": 77.02,
+        "Namaa Tourism": 79.6,
+        "Namaa Media": 77.22,
+        "Namaa Legal": 84.41,
+        "Navid General Knowledge": 77.61,
+        "Navid Web Search": 95.71,
+        "Navid Islamic Studies": 96.14,
+        "Downloads Last Month": 1274,
         "Num Likes": 7,
-        "License": "apache-2.0",
-        "Precision": "F32",
+        "License": "apache-2.0"
+    },
+    {
+        "Model": "NAMAA-Space/Namaa-ARA-Reranker-V1",
+        "Revision": "main",
+        "Precision": "f32",
         "Task": "Reranker",
-        "Revision": "main"
+        "Average Score": 85.82,
+        "Context Length": 8192,
+        "Embedding Dimension": 1024,
+        "Model Size (MB)": 2165.81,
+        "Number of Parameters (Billions)": 0.568,
+        "Namaa Global Knowledge": 80.18,
+        "Namaa Tourism": 77.7,
+        "Namaa Media": 79.07,
+        "Namaa Legal": 87.62,
+        "Navid General Knowledge": 85.19,
+        "Navid Web Search": 98.81,
+        "Navid Islamic Studies": 92.19,
+        "Downloads Last Month": 41,
+        "Num Likes": 4,
+        "License": "apache-2.0"
     },
     {
-        "Model": "cross-encoder/ms-marco-MiniLM-L-12-v2",
-        "Overall Score": 60.54,
-        "Embedding Dimensions": 384,
-        "Model Parameters (in Millions)": 33.4,
-        "Downloads Last Month": 777681,
-        "MRR": 41.69,
-        "nDCG": 67.62,
-        "MAP": 72.32,
-        "Num Likes": 68,
-        "License": "apache-2.0",
-        "Precision": "I64",
+        "Model": "OmarAlsaabi/e5-base-mlqa-finetuned-arabic-for-rag",
+        "Revision": "main",
+        "Precision": "f16",
+        "Task": "Reranker",
+        "Average Score": 49.99,
+        "Context Length": 512,
+        "Embedding Dimension": 768,
+        "Model Size (MB)": 530.33,
+        "Number of Parameters (Billions)": 0.278,
+        "Namaa Global Knowledge": 48.65,
+        "Namaa Tourism": 47.23,
+        "Namaa Media": 51.33,
+        "Namaa Legal": 45.23,
+        "Navid General Knowledge": 52.75,
+        "Navid Web Search": 52.65,
+        "Navid Islamic Studies": 52.11,
+        "Downloads Last Month": 79,
+        "Num Likes": 5,
+        "License": "N/A"
+    },
+    {
+        "Model": "OmarAlsaabi/e5-base-mlqa-finetuned-arabic-for-rag",
+        "Revision": "main",
+        "Precision": "f32",
         "Task": "Reranker",
-        "Revision": "main"
+        "Average Score": 51.83,
+        "Context Length": 512,
+        "Embedding Dimension": 768,
+        "Model Size (MB)": 1060.65,
+        "Number of Parameters (Billions)": 0.278,
+        "Namaa Global Knowledge": 50.2,
+        "Namaa Tourism": 53.47,
+        "Namaa Media": 51.93,
+        "Namaa Legal": 51.72,
+        "Navid General Knowledge": 55.01,
+        "Navid Web Search": 47.75,
+        "Navid Islamic Studies": 52.7,
+        "Downloads Last Month": 79,
+        "Num Likes": 5,
+        "License": "N/A"
     },
     {
         "Model": "Omartificial-Intelligence-Space/Arabic-MiniLM-L12-v2-all-nli-triplet",
-        "Overall Score": 58.17,
-        "Embedding Dimensions": 384,
-        "Model Parameters (in Millions)": 118.0,
-        "Downloads Last Month": 448,
-        "MRR": 42.28,
-        "nDCG": 75.63,
-        "MAP": 56.59,
+        "Revision": "main",
+        "Precision": "f32",
+        "Task": "Reranker",
+        "Average Score": 47.93,
+        "Context Length": 128,
+        "Embedding Dimension": 384,
+        "Model Size (MB)": 448.81,
+        "Number of Parameters (Billions)": 0.118,
+        "Namaa Global Knowledge": 50.18,
+        "Namaa Tourism": 49.26,
+        "Namaa Media": 48.13,
+        "Namaa Legal": 45.68,
+        "Navid General Knowledge": 43.49,
+        "Navid Web Search": 48.87,
+        "Navid Islamic Studies": 49.87,
+        "Downloads Last Month": 127,
         "Num Likes": 4,
-        "License": "apache-2.0",
-        "Precision": "F32",
+        "License": "apache-2.0"
+    },
+    {
+        "Model": "Omartificial-Intelligence-Space/Arabic-Triplet-Matryoshka-V2",
+        "Revision": "main",
+        "Precision": "f32",
         "Task": "Reranker",
-        "Revision": "main"
+        "Average Score": 49.33,
+        "Context Length": 512,
+        "Embedding Dimension": 768,
+        "Model Size (MB)": 515.72,
+        "Number of Parameters (Billions)": 0.135,
+        "Namaa Global Knowledge": 50.03,
+        "Namaa Tourism": 50.52,
+        "Namaa Media": 48.73,
+        "Namaa Legal": 50.86,
+        "Navid General Knowledge": 49.1,
+        "Navid Web Search": 49.56,
+        "Navid Islamic Studies": 46.49,
+        "Downloads Last Month": 8143,
+        "Num Likes": 10,
+        "License": "apache-2.0"
     },
     {
-        "Model": "aubmindlab/bert-base-arabert",
-        "Overall Score": 56.69,
-        "Embedding Dimensions": 768,
-        "Model Parameters (in Millions)": 136.0,
-        "Downloads Last Month": 71050,
-        "MRR": 40.16,
-        "nDCG": 71.14,
-        "MAP": 58.77,
-        "Num Likes": 27,
-        "License": "N/A",
-        "Precision": "F32",
+        "Model": "Omartificial-Intelligence-Space/Arabic-all-nli-triplet-Matryoshka",
+        "Revision": "main",
+        "Precision": "f32",
         "Task": "Reranker",
-        "Revision": "main"
+        "Average Score": 47.86,
+        "Context Length": 128,
+        "Embedding Dimension": 768,
+        "Model Size (MB)": 1060.65,
+        "Number of Parameters (Billions)": 0.278,
+        "Namaa Global Knowledge": 47.51,
+        "Namaa Tourism": 48.59,
+        "Namaa Media": 50.78,
+        "Namaa Legal": 46.82,
+        "Navid General Knowledge": 41.96,
+        "Navid Web Search": 51.73,
+        "Navid Islamic Studies": 47.65,
+        "Downloads Last Month": 291,
+        "Num Likes": 2,
+        "License": "apache-2.0"
     },
     {
-        "Model": "OrdalieTech/Solon-embeddings-large-0.1",
-        "Overall Score": 54.73,
-        "Embedding Dimensions": 1024,
-        "Model Parameters (in Millions)": 560.0,
-        "Downloads Last Month": 23830,
-        "MRR": 32.59,
-        "nDCG": 60.18,
-        "MAP": 71.41,
-        "Num Likes": 47,
-        "License": "mit",
-        "Precision": "F32",
+        "Model": "Omartificial-Intelligence-Space/Arabic-labse-Matryoshka",
+        "Revision": "main",
+        "Precision": "f32",
         "Task": "Reranker",
-        "Revision": "main"
+        "Average Score": 49.74,
+        "Context Length": 256,
+        "Embedding Dimension": 768,
+        "Model Size (MB)": 1798.7,
+        "Number of Parameters (Billions)": 0.472,
+        "Namaa Global Knowledge": 48.34,
+        "Namaa Tourism": 48.06,
+        "Namaa Media": 49.59,
+        "Namaa Legal": 44.44,
+        "Navid General Knowledge": 60.29,
+        "Navid Web Search": 46.9,
+        "Navid Islamic Studies": 50.54,
+        "Downloads Last Month": 215,
+        "Num Likes": 2,
+        "License": "apache-2.0"
     },
     {
-        "Model": "silma-ai/silma-embeddding-matryoshka-v0.1",
-        "Overall Score": 54.16,
-        "Embedding Dimensions": 768,
-        "Model Parameters (in Millions)": 135.0,
-        "Downloads Last Month": 957,
-        "MRR": 35.6,
-        "nDCG": 63.25,
-        "MAP": 63.64,
-        "Num Likes": 9,
-        "License": "apache-2.0",
-        "Precision": "F32",
+        "Model": "OrdalieTech/Solon-embeddings-large-0.1",
+        "Revision": "main",
+        "Precision": "f32",
         "Task": "Reranker",
-        "Revision": "main"
+        "Average Score": 49.55,
+        "Context Length": 512,
+        "Embedding Dimension": 1024,
+        "Model Size (MB)": 2135.81,
+        "Number of Parameters (Billions)": 0.56,
+        "Namaa Global Knowledge": 48.29,
+        "Namaa Tourism": 51.34,
+        "Namaa Media": 49.63,
+        "Namaa Legal": 43.93,
+        "Navid General Knowledge": 49.86,
+        "Navid Web Search": 50.01,
+        "Navid Islamic Studies": 53.8,
+        "Downloads Last Month": 7825,
+        "Num Likes": 50,
+        "License": "mit"
     },
     {
-        "Model": "sentence-transformers/LaBSE",
-        "Overall Score": 53.58,
-        "Embedding Dimensions": 768,
-        "Model Parameters (in Millions)": 471.0,
-        "Downloads Last Month": 745051,
-        "MRR": 32.9,
-        "nDCG": 67.82,
-        "MAP": 60.02,
-        "Num Likes": 242,
-        "License": "apache-2.0",
-        "Precision": "F32",
+        "Model": "Snowflake/snowflake-arctic-embed-l-v2.0",
+        "Revision": "main",
+        "Precision": "f32",
         "Task": "Reranker",
-        "Revision": "main"
+        "Average Score": 50.7,
+        "Context Length": 8192,
+        "Embedding Dimension": 1024,
+        "Model Size (MB)": 2165.81,
+        "Number of Parameters (Billions)": 0.568,
+        "Namaa Global Knowledge": 47.45,
+        "Namaa Tourism": 48.7,
+        "Namaa Media": 51.26,
+        "Namaa Legal": 49.66,
+        "Navid General Knowledge": 49.72,
+        "Navid Web Search": 55.93,
+        "Navid Islamic Studies": 52.16,
+        "Downloads Last Month": 117067,
+        "Num Likes": 148,
+        "License": "apache-2.0"
     },
     {
-        "Model": "intfloat/multilingual-e5-large-instruct",
-        "Overall Score": 53.39,
-        "Embedding Dimensions": 1024,
-        "Model Parameters (in Millions)": 560.0,
-        "Downloads Last Month": 391080,
-        "MRR": 48.35,
-        "nDCG": 56.06,
-        "MAP": 55.78,
-        "Num Likes": 288,
-        "License": "mit",
-        "Precision": "F16",
+        "Model": "anondeb/arabertv02_reranker_2021",
+        "Revision": "main",
+        "Precision": "f32",
         "Task": "Reranker",
-        "Revision": "main"
+        "Average Score": 82.28,
+        "Context Length": 512,
+        "Embedding Dimension": 768,
+        "Model Size (MB)": 515.72,
+        "Number of Parameters (Billions)": 0.135,
+        "Namaa Global Knowledge": 76.43,
+        "Namaa Tourism": 77.25,
+        "Namaa Media": 75.3,
+        "Namaa Legal": 82.63,
+        "Navid General Knowledge": 75.05,
+        "Navid Web Search": 94.11,
+        "Navid Islamic Studies": 95.18,
+        "Downloads Last Month": 23,
+        "Num Likes": 0,
+        "License": "cc-by-nc-4.0"
     },
     {
-        "Model": "Omartificial-Intelligence-Space/Arabic-all-nli-triplet-Matryoshka",
-        "Overall Score": 52.16,
-        "Embedding Dimensions": 768,
-        "Model Parameters (in Millions)": 135.0,
-        "Downloads Last Month": 503,
-        "MRR": 40.58,
-        "nDCG": 54.7,
-        "MAP": 61.2,
-        "Num Likes": 2,
-        "License": "apache-2.0",
-        "Precision": "F32",
+        "Model": "asafaya/bert-base-arabic",
+        "Revision": "main",
+        "Precision": "f32",
         "Task": "Reranker",
-        "Revision": "main"
+        "Average Score": 68.89,
+        "Context Length": 512,
+        "Embedding Dimension": 768,
+        "Model Size (MB)": 421.97,
+        "Number of Parameters (Billions)": 0.111,
+        "Namaa Global Knowledge": 63.6,
+        "Namaa Tourism": 59.54,
+        "Namaa Media": 61.14,
+        "Namaa Legal": 72.6,
+        "Navid General Knowledge": 63.27,
+        "Navid Web Search": 86.84,
+        "Navid Islamic Studies": 75.25,
+        "Downloads Last Month": 10439,
+        "Num Likes": 38,
+        "License": "N/A"
     },
     {
-        "Model": "Snowflake/snowflake-arctic-embed-l-v2.0",
-        "Overall Score": 51.61,
-        "Embedding Dimensions": 1024,
-        "Model Parameters (in Millions)": 568.0,
-        "Downloads Last Month": 86764,
-        "MRR": 32.16,
-        "nDCG": 58.63,
-        "MAP": 64.05,
-        "Num Likes": 111,
-        "License": "apache-2.0",
-        "Precision": "F32",
+        "Model": "aubmindlab/bert-base-arabert",
+        "Revision": "main",
+        "Precision": "f32",
         "Task": "Reranker",
-        "Revision": "main"
+        "Average Score": 56.75,
+        "Context Length": 512,
+        "Embedding Dimension": 768,
+        "Model Size (MB)": 515.72,
+        "Number of Parameters (Billions)": 0.135,
+        "Namaa Global Knowledge": 51.19,
+        "Namaa Tourism": 50.61,
+        "Namaa Media": 53.32,
+        "Namaa Legal": 49.75,
+        "Navid General Knowledge": 58.99,
+        "Navid Web Search": 64.7,
+        "Navid Islamic Studies": 68.72,
+        "Downloads Last Month": 50761,
+        "Num Likes": 29,
+        "License": "N/A"
     },
     {
         "Model": "aubmindlab/bert-large-arabertv2",
-        "Overall Score": 50.84,
-        "Embedding Dimensions": 1024,
-        "Model Parameters (in Millions)": 371.0,
-        "Downloads Last Month": 1352,
-        "MRR": 27.84,
-        "nDCG": 53.42,
-        "MAP": 71.27,
+        "Revision": "main",
+        "Precision": "i64",
+        "Task": "Reranker",
+        "Average Score": 48.68,
+        "Context Length": 512,
+        "Embedding Dimension": 1024,
+        "Model Size (MB)": 1409.24,
+        "Number of Parameters (Billions)": 0.369,
+        "Namaa Global Knowledge": 46.56,
+        "Namaa Tourism": 46.53,
+        "Namaa Media": 46.39,
+        "Namaa Legal": 45.89,
+        "Navid General Knowledge": 45.88,
+        "Navid Web Search": 63.33,
+        "Navid Islamic Studies": 46.16,
+        "Downloads Last Month": 2059,
         "Num Likes": 11,
-        "License": "N/A",
-        "Precision": "I64",
+        "License": "N/A"
+    },
+    {
+        "Model": "colbert-ir/colbertv2.0",
+        "Revision": "main",
+        "Precision": "i64",
         "Task": "Reranker",
-        "Revision": "main"
+        "Average Score": 52.44,
+        "Context Length": 512,
+        "Embedding Dimension": 768,
+        "Model Size (MB)": 417.64,
+        "Number of Parameters (Billions)": 0.109,
+        "Namaa Global Knowledge": 47.92,
+        "Namaa Tourism": 51.21,
+        "Namaa Media": 50.97,
+        "Namaa Legal": 44.7,
+        "Navid General Knowledge": 76.33,
+        "Navid Web Search": 49.91,
+        "Navid Islamic Studies": 46.06,
+        "Downloads Last Month": 1409633,
+        "Num Likes": 248,
+        "License": "mit"
     },
     {
-        "Model": "Lajavaness/bilingual-embedding-large",
-        "Overall Score": 49.91,
-        "Embedding Dimensions": 1024,
-        "Model Parameters (in Millions)": 560.0,
-        "Downloads Last Month": 20987,
-        "MRR": 33.93,
-        "nDCG": 57.33,
-        "MAP": 58.46,
-        "Num Likes": 17,
-        "License": "apache-2.0",
-        "Precision": "F32",
-        "Task": "Reranker",
-        "Revision": "main"
-    },
-    {
-        "Model": "jinaai/jina-embeddings-v3",
-        "Overall Score": 49.44,
-        "Embedding Dimensions": 4096,
-        "Model Parameters (in Millions)": 572.0,
-        "Downloads Last Month": 1523322,
-        "MRR": 32.03,
-        "nDCG": 60.41,
-        "MAP": 55.87,
-        "Num Likes": 726,
-        "License": "cc-by-nc-4.0",
-        "Precision": "BF16",
-        "Task": "Reranker",
-        "Revision": "main"
+        "Model": "cross-encoder/ms-marco-MiniLM-L-12-v2",
+        "Revision": "main",
+        "Precision": "i64",
+        "Task": "Reranker",
+        "Average Score": 59.62,
+        "Context Length": 512,
+        "Embedding Dimension": 384,
+        "Model Size (MB)": 127.26,
+        "Number of Parameters (Billions)": 0.033,
+        "Namaa Global Knowledge": 62.81,
+        "Namaa Tourism": 59.16,
+        "Namaa Media": 60.86,
+        "Namaa Legal": 68.86,
+        "Navid General Knowledge": 60.42,
+        "Navid Web Search": 51.06,
+        "Navid Islamic Studies": 54.17,
+        "Downloads Last Month": 498422,
+        "Num Likes": 70,
+        "License": "apache-2.0"
     },
     {
-        "Model": "Omartificial-Intelligence-Space/Arabic-labse-Matryoshka",
-        "Overall Score": 48.76,
-        "Embedding Dimensions": 768,
-        "Model Parameters (in Millions)": 471.0,
-        "Downloads Last Month": 566,
-        "MRR": 32.71,
-        "nDCG": 56.6,
-        "MAP": 56.97,
-        "Num Likes": 2,
-        "License": "apache-2.0",
-        "Precision": "F32",
+        "Model": "intfloat/multilingual-e5-large-instruct",
+        "Revision": "main",
+        "Precision": "f16",
+        "Task": "Reranker",
+        "Average Score": 53.46,
+        "Context Length": 512,
+        "Embedding Dimension": 1024,
+        "Model Size (MB)": 1067.91,
+        "Number of Parameters (Billions)": 0.56,
+        "Namaa Global Knowledge": 52.6,
+        "Namaa Tourism": 51.09,
+        "Namaa Media": 52.51,
+        "Namaa Legal": 50.97,
+        "Navid General Knowledge": 67.73,
+        "Navid Web Search": 49.48,
+        "Navid Islamic Studies": 49.83,
+        "Downloads Last Month": 995915,
+        "Num Likes": 416,
+        "License": "mit"
+    },
+    {
+        "Model": "oddadmix/arabic-reranker-v1",
+        "Revision": "main",
+        "Precision": "f32",
+        "Task": "Reranker",
+        "Average Score": 79.93,
+        "Context Length": 512,
+        "Embedding Dimension": 768,
+        "Model Size (MB)": 515.72,
+        "Number of Parameters (Billions)": 0.135,
+        "Namaa Global Knowledge": 74.08,
+        "Namaa Tourism": 72.28,
+        "Namaa Media": 70.31,
+        "Namaa Legal": 78.21,
+        "Navid General Knowledge": 85.0,
+        "Navid Web Search": 85.65,
+        "Navid Islamic Studies": 93.98,
+        "Downloads Last Month": 23,
+        "Num Likes": 1,
+        "License": "N/A"
+    },
+    {
+        "Model": "omarelshehy/Arabic-Retrieval-v1.0",
+        "Revision": "main",
+        "Precision": "f32",
+        "Task": "Reranker",
+        "Average Score": 43.7,
+        "Context Length": 512,
+        "Embedding Dimension": 768,
+        "Model Size (MB)": 515.73,
+        "Number of Parameters (Billions)": 0.135,
+        "Namaa Global Knowledge": 47.98,
+        "Namaa Tourism": 51.39,
+        "Namaa Media": 47.91,
+        "Namaa Legal": 50.77,
+        "Navid General Knowledge": 50.42,
+        "Navid Web Search": 29.07,
+        "Navid Islamic Studies": 28.39,
+        "Downloads Last Month": 174,
+        "Num Likes": 1,
+        "License": "apache-2.0"
+    },
+    {
+        "Model": "sentence-transformers/LaBSE",
+        "Revision": "main",
+        "Precision": "i64",
         "Task": "Reranker",
-        "Revision": "main"
+        "Average Score": 49.88,
+        "Context Length": 256,
+        "Embedding Dimension": 768,
+        "Model Size (MB)": 1798.7,
+        "Number of Parameters (Billions)": 0.472,
+        "Namaa Global Knowledge": 51.04,
+        "Namaa Tourism": 49.33,
+        "Namaa Media": 49.27,
+        "Namaa Legal": 45.63,
+        "Navid General Knowledge": 52.54,
+        "Navid Web Search": 49.96,
+        "Navid Islamic Studies": 51.39,
+        "Downloads Last Month": 558352,
+        "Num Likes": 258,
+        "License": "apache-2.0"
+    },
+    {
+        "Model": "silma-ai/silma-embeddding-matryoshka-v0.1",
+        "Revision": "main",
+        "Precision": "f32",
+        "Task": "Reranker",
+        "Average Score": 44.05,
+        "Context Length": 512,
+        "Embedding Dimension": 768,
+        "Model Size (MB)": 515.72,
+        "Number of Parameters (Billions)": 0.135,
+        "Namaa Global Knowledge": 47.17,
+        "Namaa Tourism": 46.42,
+        "Namaa Media": 47.97,
+        "Namaa Legal": 52.81,
+        "Navid General Knowledge": 54.47,
+        "Navid Web Search": 30.82,
+        "Navid Islamic Studies": 28.66,
+        "Downloads Last Month": 405,
+        "Num Likes": 11,
+        "License": "apache-2.0"
     }
 ]

results/retrieval_results.json

@@ -1,162 +1,114 @@
 [
     {
-        "Model": "Omartificial-Intelligence-Space/Arabic-MiniLM-L12-v2-all-nli-triplet",
-        "Max Tokens": 128,
+        "Model": "Alibaba-NLP/gte-multilingual-base",
         "Revision": "main",
-        "Precision": "F32",
+        "Precision": "f16",
         "Task": "Retriever",
-        "Embedding Dimension": 384,
-        "Model Size (MB)": 448.81,
-        "Web Search Dataset (MRR)": 46.0,
-        "Web Search Dataset (Recall@5)": 56.19,
-        "Web Search Dataset (nDCG@k=None)": 55.34,
-        "Web Search Dataset (Overall Score)": 52.51,
-        "Downloads Last Month": 448,
-        "Num Likes": 4,
+        "Average Score": 61.02,
+        "Context Length": 8192,
+        "Embedding Dimension": 768,
+        "Model Size (MB)": 582.44,
+        "Number of Parameters (Billions)": 0.305,
+        "Web Search Dataset": 80.2,
+        "Islamic Knowledge Dataset": 41.84,
+        "Downloads Last Month": 1340501,
+        "Num Likes": 233,
         "License": "apache-2.0"
     },
     {
-        "Model": "Omartificial-Intelligence-Space/Arabic-labse-Matryoshka",
-        "Max Tokens": 256,
+        "Model": "NAMAA-Space/AraModernBert-Base-STS",
         "Revision": "main",
-        "Precision": "F32",
+        "Precision": "f32",
         "Task": "Retriever",
+        "Average Score": 49.99,
+        "Context Length": 512,
         "Embedding Dimension": 768,
-        "Model Size (MB)": 1798.7,
-        "Web Search Dataset (MRR)": 56.96,
-        "Web Search Dataset (Recall@5)": 70.05,
-        "Web Search Dataset (nDCG@k=None)": 65.27,
-        "Web Search Dataset (Overall Score)": 64.09,
-        "Downloads Last Month": 566,
-        "Num Likes": 2,
+        "Model Size (MB)": 568.19,
+        "Number of Parameters (Billions)": 0.149,
+        "Web Search Dataset": 37.9,
+        "Islamic Knowledge Dataset": 62.08,
+        "Downloads Last Month": 205,
+        "Num Likes": 6,
         "License": "apache-2.0"
     },
     {
-        "Model": "intfloat/multilingual-e5-large-instruct",
-        "Max Tokens": 512,
+        "Model": "Omartificial-Intelligence-Space/Arabert-all-nli-triplet-Matryoshka",
         "Revision": "main",
-        "Precision": "F16",
-        "Task": "Retriever",
-        "Embedding Dimension": 1024,
-        "Model Size (MB)": 1067.91,
-        "Web Search Dataset (MRR)": 65.26,
-        "Web Search Dataset (Recall@5)": 74.14,
-        "Web Search Dataset (nDCG@k=None)": 71.66,
-        "Web Search Dataset (Overall Score)": 70.35,
-        "Downloads Last Month": 391080,
-        "Num Likes": 288,
-        "License": "mit"
-    },
-    {
-        "Model": "omarelshehy/Arabic-Retrieval-v1.0",
-        "Max Tokens": 512,
-        "Revision": "main",
-        "Precision": "F32",
+        "Precision": "f32",
         "Task": "Retriever",
+        "Average Score": 42.54,
+        "Context Length": 512,
         "Embedding Dimension": 768,
-        "Model Size (MB)": 515.73,
-        "Web Search Dataset (MRR)": 55.32,
-        "Web Search Dataset (Recall@5)": 64.76,
-        "Web Search Dataset (nDCG@k=None)": 63.48,
-        "Web Search Dataset (Overall Score)": 61.19,
-        "Downloads Last Month": 194,
-        "Num Likes": 0,
+        "Model Size (MB)": 515.72,
+        "Number of Parameters (Billions)": 0.135,
+        "Web Search Dataset": 44.49,
+        "Islamic Knowledge Dataset": 40.59,
+        "Downloads Last Month": 697,
+        "Num Likes": 10,
         "License": "apache-2.0"
     },
     {
-        "Model": "omarelshehy/Arabic-STS-Matryoshka-V2",
-        "Max Tokens": 512,
+        "Model": "Omartificial-Intelligence-Space/Arabic-Triplet-Matryoshka-V2",
         "Revision": "main",
-        "Precision": "F32",
+        "Precision": "f32",
         "Task": "Retriever",
+        "Average Score": 55.14,
+        "Context Length": 512,
         "Embedding Dimension": 768,
         "Model Size (MB)": 515.72,
-        "Web Search Dataset (MRR)": 41.83,
-        "Web Search Dataset (Recall@5)": 49.21,
-        "Web Search Dataset (nDCG@k=None)": 50.71,
-        "Web Search Dataset (Overall Score)": 47.25,
-        "Downloads Last Month": 264,
-        "Num Likes": 1,
-        "License": "N/A"
-    },
-    {
-        "Model": "omarelshehy/Arabic-STS-Matryoshka",
-        "Max Tokens": 512,
-        "Revision": "main",
-        "Precision": "F32",
-        "Task": "Retriever",
-        "Embedding Dimension": 1024,
-        "Model Size (MB)": 2135.81,
-        "Web Search Dataset (MRR)": 63.2,
-        "Web Search Dataset (Recall@5)": 74.41,
-        "Web Search Dataset (nDCG@k=None)": 70.43,
-        "Web Search Dataset (Overall Score)": 69.35,
-        "Downloads Last Month": 167,
-        "Num Likes": 2,
+        "Number of Parameters (Billions)": 0.135,
+        "Web Search Dataset": 50.93,
+        "Islamic Knowledge Dataset": 59.35,
+        "Downloads Last Month": 8143,
+        "Num Likes": 10,
         "License": "apache-2.0"
     },
     {
-        "Model": "omarelshehy/arabic-english-sts-matryoshka-v2.0",
-        "Max Tokens": 512,
-        "Revision": "main",
-        "Precision": "F32",
-        "Task": "Retriever",
-        "Embedding Dimension": 1024,
-        "Model Size (MB)": 2135.81,
-        "Web Search Dataset (MRR)": 56.02,
-        "Web Search Dataset (Recall@5)": 67.85,
-        "Web Search Dataset (nDCG@k=None)": 64.47,
-        "Web Search Dataset (Overall Score)": 62.78,
-        "Downloads Last Month": 324,
-        "Num Likes": 1,
-        "License": "N/A"
-    },
-    {
-        "Model": "omarelshehy/arabic-english-sts-matryoshka",
-        "Max Tokens": 512,
+        "Model": "Omartificial-Intelligence-Space/GATE-AraBert-v1",
         "Revision": "main",
-        "Precision": "F32",
+        "Precision": "f16",
         "Task": "Retriever",
-        "Embedding Dimension": 1024,
-        "Model Size (MB)": 2135.81,
-        "Web Search Dataset (MRR)": 51.5,
-        "Web Search Dataset (Recall@5)": 62.91,
-        "Web Search Dataset (nDCG@k=None)": 60.55,
-        "Web Search Dataset (Overall Score)": 58.32,
-        "Downloads Last Month": 295,
-        "Num Likes": 0,
+        "Average Score": 53.53,
+        "Context Length": 512,
+        "Embedding Dimension": 768,
+        "Model Size (MB)": 257.86,
+        "Number of Parameters (Billions)": 0.135,
+        "Web Search Dataset": 50.97,
+        "Islamic Knowledge Dataset": 56.09,
+        "Downloads Last Month": 3885,
+        "Num Likes": 12,
         "License": "apache-2.0"
     },
     {
-        "Model": "silma-ai/silma-embeddding-matryoshka-v0.1",
-        "Max Tokens": 512,
+        "Model": "mohamed2811/Muffakir_Embedding",
         "Revision": "main",
-        "Precision": "F32",
+        "Precision": "f32",
         "Task": "Retriever",
+        "Average Score": 60.03,
+        "Context Length": 512,
         "Embedding Dimension": 768,
         "Model Size (MB)": 515.72,
-        "Web Search Dataset (MRR)": 45.54,
-        "Web Search Dataset (Recall@5)": 56.5,
-        "Web Search Dataset (nDCG@k=None)": 55.32,
-        "Web Search Dataset (Overall Score)": 52.46,
-        "Downloads Last Month": 957,
-        "Num Likes": 9,
-        "License": "apache-2.0"
+        "Number of Parameters (Billions)": 0.135,
+        "Web Search Dataset": 54.5,
+        "Islamic Knowledge Dataset": 65.56,
+        "Downloads Last Month": 615,
+        "Num Likes": 1,
+        "License": "N/A"
     },
     {
-        "Model": "silma-ai/silma-embeddding-sts-v0.1",
-        "Max Tokens": 512,
+        "Model": "omarelshehy/Arabic-STS-Matryoshka-V2",
         "Revision": "main",
-        "Precision": "F32",
+        "Precision": "f16",
         "Task": "Retriever",
+        "Average Score": 52.38,
+        "Context Length": 512,
         "Embedding Dimension": 768,
-        "Model Size (MB)": 515.72,
-        "Web Search Dataset (MRR)": 47.28,
-        "Web Search Dataset (Recall@5)": 57.97,
-        "Web Search Dataset (nDCG@k=None)": 56.72,
-        "Web Search Dataset (Overall Score)": 53.99,
-        "Downloads Last Month": 1392,
-        "Num Likes": 4,
-        "License": "apache-2.0"
+        "Model Size (MB)": 257.86,
+        "Number of Parameters (Billions)": 0.135,
+        "Web Search Dataset": 47.25,
+        "Islamic Knowledge Dataset": 57.5,
+        "Downloads Last Month": 263,
+        "Num Likes": 1,
+        "License": "N/A"
     }
 ]

retrieval_leaderboard.py

@@ -0,0 +1,87 @@
+from pathlib import Path
+from utils import load_json_results
+from leaderboard_tab import search_leaderboard, update_columns_to_show, create_leaderboard_tab
+
+# Constants
+RETRIEVAL_ABOUT_SECTION = """
+## About Retrieval Evaluation
+
+The retrieval evaluation assesses a model's ability to find and retrieve relevant information from a large corpus of Arabic text. Models are evaluated on:
+
+### Web Search Dataset Metrics
+- **MRR (Mean Reciprocal Rank)**: Measures the ranking quality by focusing on the position of the first relevant result
+- **nDCG (Normalized Discounted Cumulative Gain)**: Evaluates the ranking quality considering all relevant results
+- **Recall@5**: Measures the proportion of relevant documents found in the top 5 results
+- **Overall Score**: Combined score calculated as the average of MRR, nDCG, and Recall@5
+
+### Model Requirements
+- Must support Arabic text embeddings
+- Should handle queries of at least 512 tokens 
+- Must work with `sentence-transformers` library
+
+### Evaluation Process
+1. Models process Arabic web search queries
+2. Retrieved documents are evaluated using:
+   - MRR for first relevant result positioning
+   - nDCG for overall ranking quality
+   - Recall@5 for top results accuracy
+3. Metrics are averaged to calculate the overall score
+4. Models are ranked based on their overall performance
+
+### How to Prepare Your Model
+- Ensure your model is publicly available on HuggingFace Hub (We don't support private model evaluations yet)
+- Model should output fixed-dimension embeddings for text
+- Support batch processing for efficient evaluation (this is default if you use `sentence-transformers`)
+"""
+
+# Global variables
+retrieval_df = None
+
+def load_retrieval_results(prepare_for_display=False, sort_col=None, drop_cols=None):
+    dataframe_path = Path(__file__).parent / "results" / "retrieval_results.json"
+    return load_json_results(
+        dataframe_path,
+        prepare_for_display=prepare_for_display,
+        sort_col=sort_col,
+        drop_cols=drop_cols
+    )
+
+def load_retrieval_leaderboard():
+    """Load and prepare the retrieval leaderboard data"""
+    global retrieval_df
+    
+    # Prepare retrieval dataframe
+    retrieval_df = load_retrieval_results(True, "Average Score", drop_cols=["Revision", "Precision", "Task"])
+    retrieval_df.insert(0, "Rank", range(1, 1 + len(retrieval_df)))
+    
+    return retrieval_df
+
+def retrieval_search_leaderboard(model_name, columns_to_show):
+    """Search function for retrieval leaderboard"""
+    return search_leaderboard(retrieval_df, model_name, columns_to_show)
+
+def update_retrieval_columns_to_show(columns_to_show):
+    """Update displayed columns for retrieval leaderboard"""
+    return update_columns_to_show(retrieval_df, columns_to_show)
+
+def create_retrieval_tab():
+    """Create the complete retrieval leaderboard tab"""
+    global retrieval_df
+    
+    # Load data if not already loaded
+    if retrieval_df is None:
+        retrieval_df = load_retrieval_leaderboard()
+    
+    # Define default columns to show
+    default_columns = ["Rank", "Model", "Average Score", "Model Size (MB)", "Context Length",
+                       "Embedding Dimension", "Web Search Dataset", "Islamic Knowledge Dataset"]
+    
+    # Create and return the tab
+    return create_leaderboard_tab(
+        df=retrieval_df,
+        initial_columns_to_show=default_columns,
+        search_function=retrieval_search_leaderboard,
+        update_function=update_retrieval_columns_to_show,
+        about_section=RETRIEVAL_ABOUT_SECTION,
+        task_type="Retriever"
+    )

utils.py

@@ -12,8 +12,11 @@ DATASET_REPO_ID = f"{OWNER}/requests-dataset"
 
 results_dir = Path(__file__).parent / "results"
 
-# Cache the HF token to avoid multiple os.environ lookups.
-HF_TOKEN = os.environ.get('HF_TOKEN', None)
+# Replace the current HF_TOKEN line with this to add a helpful error message if token is missing
+HF_TOKEN = os.environ.get('HF_TOKEN')
+if not HF_TOKEN:
+    print("Warning: HF_TOKEN environment variable not set. API operations requiring authentication will fail.")
+    HF_TOKEN = None
 
 # Add a helper to load JSON results with optional formatting.
 def load_json_results(file_path: Path, prepare_for_display=False, sort_col=None, drop_cols=None):
@@ -30,24 +33,6 @@ def load_json_results(file_path: Path, prepare_for_display=False, sort_col=None,
             df.sort_values(sort_col, ascending=False, inplace=True)
     return df
 
-def load_retrieval_results(prepare_for_display=False, sort_col=None, drop_cols=None):
-    dataframe_path = results_dir / "retrieval_results.json"
-    return load_json_results(
-        dataframe_path,
-        prepare_for_display=prepare_for_display,
-        sort_col=sort_col,
-        drop_cols=drop_cols
-    )
-
-def load_reranking_results(prepare_for_display=False, sort_col=None, drop_cols=None):
-    dataframe_path = results_dir / "reranking_results.json"
-    return load_json_results(
-        dataframe_path,
-        prepare_for_display=prepare_for_display,
-        sort_col=sort_col,
-        drop_cols=drop_cols
-    )
-
 def get_model_info(model_id, verbose=False):
     model_info = api.model_info(model_id)
     num_downloads = model_info.downloads
@@ -71,16 +56,12 @@ def fetch_model_information(model_name):
         return
     return gr.update(choices=supported_precisions, value=supported_precisions[0]), license, num_parameters, num_downloads, num_likes
 
-def submit_model(model_name, revision, precision, params, license, task):
-    # Load existing evaluations
-    if task == "Retriever":
-        df = load_retrieval_results()
-    elif task == "Reranker":
-        df = load_reranking_results()
-    else:
-        return "Task is not supported 🤷‍♂️"
-    
-    existing_models_results = df[['Model', 'Revision', 'Precision', 'Task']]
+def submit_model(model_name, revision, precision, params, license, task, pending_gradio_df):
+    try:
+        if float(params) > 5000:
+            return "Model size should be less than 5000 million parameters (5 billion) 👀", pending_gradio_df
+    except ValueError:
+        gr.Error("The parameter count is not present or is not a number. Please make sure its available and its correct"),
 
     # Handle 'Missing' precision
     if precision == 'Missing':
@@ -92,14 +73,6 @@ def submit_model(model_name, revision, precision, params, license, task):
     df_pending = load_requests('pending')
     df_finished = load_requests('finished')
 
-    # Check if model is already evaluated
-    model_exists_in_results = ((existing_models_results['Model'] == model_name) &
-                               (existing_models_results['Revision'] == revision) &
-                               (existing_models_results['Precision'] == precision.capitalize()) &
-                               (existing_models_results['Task'] == task)).any()
-    if model_exists_in_results:
-        return f"Model {model_name} has already been evaluated as a {task} 🎉"
-
     # Check if model is in pending requests
     if not df_pending.empty:
         existing_models_pending = df_pending[['model_name', 'revision', 'precision', 'task']]
@@ -108,7 +81,7 @@ def submit_model(model_name, revision, precision, params, license, task):
                                    (existing_models_pending['precision'] == precision.capitalize()) &
                                       (existing_models_pending['task'] == task)).any()
         if model_exists_in_pending:
-            return f"Model {model_name} is already in the evaluation queue as a {task} 🚀"
+            return f"Model {model_name} is already in the evaluation queue as a {task} 🚀", pending_gradio_df
 
     # Check if model is in finished requests
     if not df_finished.empty:
@@ -267,11 +240,6 @@ def submit_gradio_module(task_type):
         inputs=[model_name_input],
         outputs=fetch_outputs
     )
-    submit_button.click(
-        submit_model,
-        inputs=[model_name_input, revision_input, precision_input, params_input, license_input, var],
-        outputs=submission_result
-    )
     
     # Load pending, finished, and failed requests
     df_pending = load_requests('pending', task_type)
@@ -282,9 +250,10 @@ def submit_gradio_module(task_type):
     gr.Markdown("## Evaluation Status")
     with gr.Accordion(f"Pending Evaluations ({len(df_pending)})", open=False):
         if not df_pending.empty:
-            gr.Dataframe(df_pending)
+            pending_gradio_df = gr.Dataframe(df_pending)
         else:
             gr.Markdown("No pending evaluations.")
+            pending_gradio_df = None
     with gr.Accordion(f"Finished Evaluations ({len(df_finished)})", open=False):
         if not df_finished.empty:
             gr.Dataframe(df_finished)
@@ -295,3 +264,9 @@ def submit_gradio_module(task_type):
             gr.Dataframe(df_failed)
         else:
             gr.Markdown("No failed evaluations.")
+
+    submit_button.click(
+        submit_model,
+        inputs=[model_name_input, revision_input, precision_input, params_input, license_input, var, pending_gradio_df],
+        outputs=[submission_result, pending_gradio_df],
+    )