import gradio as gr
import pandas as pd

# ---------------      Marqo-Ecommerce-Easy     -------------------

# Embedding Models for Marqo-Ecommerce-Easy
embedding_models = [
    '<a href="https://docs.aws.amazon.com/bedrock/latest/userguide/titan-multiemb-models.html">Amazon-Titan-MultiModal</a>', 
    '<a href="https://huggingface.co/jinaai/jina-clip-v1">Jina-V1-CLIP</a>', 
    '<a href="https://cohere.com/blog/introducing-embed-v3">Cohere-embedding-v3</a>', 
    '<a href="https://cloud.google.com/vertex-ai">GCP-Vertex</a>', 
    '<a href="https://huggingface.co/timm/ViT-SO400M-14-SigLIP">ViT-SO400M-14-SigLip</a>', 
    '<a href="https://huggingface.co/timm/ViT-B-16-SigLIP">ViT-B-16-SigLip</a>', 
    '<a href="https://huggingface.co/timm/ViT-L-16-SigLIP-384">ViT-L-16-SigLip</a>', 
    '<a href="https://huggingface.co/Marqo/marqo-ecommerce-embeddings-B">Marqo-Ecommerce-B</a>', 
    '<a href="https://huggingface.co/Marqo/marqo-ecommerce-embeddings-L">Marqo-Ecommerce-L</a>'
]

# GS-Text2Image-1m
gs_text2_image_1m_mAP = [0.694, 0.480, 0.358, 0.740, 0.792, 0.701, 0.754, 0.842, 0.879]
gs_text2_image_1m_Recall = [0.868, 0.638, 0.515, 0.910, 0.935, 0.87, 0.907, 0.961, 0.971]
gs_text2_image_1m_MRR = [0.693, 0.480, 0.358, 0.740, 0.792, 0.701, 0.754, 0.842, 0.879]
gs_text2_image_1m_ndcg = [0.733, 0.511, 0.389, 0.779, 0.825, 0.739, 0.789, 0.871, 0.901]

gs_text2_image_1m_data = {
    "Embedding Model": embedding_models,
    "mAP": gs_text2_image_1m_mAP,
    "R@10": gs_text2_image_1m_Recall,
    "MRR": gs_text2_image_1m_MRR,
    "nDCG@10": gs_text2_image_1m_ndcg
}
gs_text2_image_1m_df = pd.DataFrame(gs_text2_image_1m_data)
gs_text2_image_1m_df = pd.DataFrame(gs_text2_image_1m_df).sort_values(by="mAP", ascending=False).reset_index(drop=True)

# GS-Category2Image-1m
gs_category2_image_1m_mAP = [0.308, 0.175, 0.136, 0.417, 0.423, 0.347, 0.392, 0.479, 0.515]
gs_category2_image_1m_Precision = [0.231, 0.122, 0.110, 0.298, 0.302, 0.252, 0.281, 0.336, 0.358]
gs_category2_image_1m_MRR = [0.558, 0.369, 0.315, 0.636, 0.644, 0.594, 0.627, 0.744, 0.764]
gs_category2_image_1m_ndcg = [0.377, 0.229, 0.178, 0.481, 0.487, 0.414, 0.458, 0.558, 0.590]

gs_category2_image_1m_data = {
    "Embedding Model": embedding_models,
    "mAP": gs_category2_image_1m_mAP,
    "P@10": gs_category2_image_1m_Precision,
    "MRR": gs_category2_image_1m_MRR,
    "nDCG@10": gs_category2_image_1m_ndcg
}
gs_category2_image_1m_df = pd.DataFrame(gs_category2_image_1m_data)
gs_category2_image_1m_df = pd.DataFrame(gs_category2_image_1m_df).sort_values(by="mAP", ascending=False).reset_index(drop=True)

# AP-Text2Image-3m
ap_text2_image_3m_mAP = [0.762, 0.530, 0.433, 0.808, 0.860, 0.797, 0.842, 0.897, 0.928]
ap_text2_image_3m_Recall = [0.889, 0.699, 0.597, 0.933, 0.954, 0.917, 0.940, 0.967, 0.978]
ap_text2_image_3m_MRR = [0.763, 0.530, 0.433, 0.808, 0.860, 0.797, 0.842, 0.897, 0.928]
ap_text2_image_3m_ndcg = [0.791, 0.565, 0.465, 0.837, 0.882, 0.825, 0.865, 0.914, 0.940]

ap_text2_image_3m_data = {
    "Embedding Model": embedding_models,
    "mAP": ap_text2_image_3m_mAP,
    "R@10": ap_text2_image_3m_Recall,
    "MRR": ap_text2_image_3m_MRR,
    "nDCG@10": ap_text2_image_3m_ndcg
}
ap_text2_image_3m_df = pd.DataFrame(ap_text2_image_3m_data)
ap_text2_image_3m_df = pd.DataFrame(ap_text2_image_3m_df).sort_values(by="mAP", ascending=False).reset_index(drop=True)


# ---------------       Marqo-Ecommerce-Hard      -------------------

# Embedding Models for Marqo-Ecommerce-Hard
hard_embedding_models = [
    '<a href="https://docs.aws.amazon.com/bedrock/latest/userguide/titan-multiemb-models.html">Amazon-Titan-MultiModal</a>', 
    '<a href="https://huggingface.co/jinaai/jina-clip-v1">Jina-V1-CLIP</a>', 
    '<a href="https://huggingface.co/timm/ViT-SO400M-14-SigLIP">ViT-SO400M-14-SigLip</a>', 
    '<a href="https://huggingface.co/timm/ViT-B-16-SigLIP">ViT-B-16-SigLip</a>', 
    '<a href="https://huggingface.co/timm/ViT-L-16-SigLIP-384">ViT-L-16-SigLip</a>', 
    '<a href="https://huggingface.co/Marqo/marqo-ecommerce-embeddings-B">Marqo-Ecommerce-B</a>', 
    '<a href="https://huggingface.co/Marqo/marqo-ecommerce-embeddings-L">Marqo-Ecommerce-L</a>'
]

# GS-Text2Image-1m
hard_gs_text2_image_1m_mAP = [0.475, 0.285, 0.573, 0.476, 0.540, 0.623, 0.682]
hard_gs_text2_image_1m_Recall = [0.648, 0.402, 0.763, 0.660, 0.722, 0.832, 0.878]
hard_gs_text2_image_1m_MRR = [0.475, 0.285, 0.574, 0.477, 0.540, 0.624, 0.683]
hard_gs_text2_image_1m_ndcg = [0.509, 0.306, 0.613, 0.513, 0.577, 0.668, 0.726]

hard_gs_text2_image_1m_data = {
    "Embedding Model": hard_embedding_models,
    "mAP": hard_gs_text2_image_1m_mAP,
    "R@10": hard_gs_text2_image_1m_Recall,
    "MRR": hard_gs_text2_image_1m_MRR,
    "nDCG@10": hard_gs_text2_image_1m_ndcg
}
hard_gs_text2_image_1m_df = pd.DataFrame(hard_gs_text2_image_1m_data)
hard_gs_text2_image_1m_df = pd.DataFrame(hard_gs_text2_image_1m_df).sort_values(by="mAP", ascending=False).reset_index(drop=True)

# GS-Category2Image-1m
hard_gs_category2_image_1m_mAP = [0.246, 0.123, 0.352, 0.277, 0.324, 0.423, 0.463]
hard_gs_category2_image_1m_Precision = [0.429, 0.275, 0.516, 0.458, 0.497, 0.629, 0.652]
hard_gs_category2_image_1m_MRR = [0.642, 0.504, 0.707, 0.660, 0.687, 0.810, 0.822]
hard_gs_category2_image_1m_ndcg = [0.446, 0.294, 0.529, 0.473, 0.509, 0.644, 0.666]

hard_gs_category2_image_1m_data = {
    "Embedding Model": hard_embedding_models,
    "mAP": hard_gs_category2_image_1m_mAP,
    "P@10": hard_gs_category2_image_1m_Precision,
    "MRR": hard_gs_category2_image_1m_MRR,
    "nDCG@10": hard_gs_category2_image_1m_ndcg
}
hard_gs_category2_image_1m_df = pd.DataFrame(hard_gs_category2_image_1m_data)
hard_gs_category2_image_1m_df = pd.DataFrame(hard_gs_category2_image_1m_df).sort_values(by="mAP", ascending=False).reset_index(drop=True)

# AP-Text2Image-3m
hard_ap_text2_image_3m_mAP = [0.456, 0.265, 0.560, 0.480, 0.544, 0.592, 0.658]
hard_ap_text2_image_3m_Recall = [0.627, 0.378, 0.742, 0.650, 0.715, 0.795, 0.854]
hard_ap_text2_image_3m_MRR = [0.457, 0.266, 0.564, 0.484, 0.548, 0.597, 0.663]
hard_ap_text2_image_3m_ndcg = [0.491, 0.285, 0.599, 0.515, 0.580, 0.637, 0.703]

hard_ap_text2_image_3m_data = {
    "Embedding Model": hard_embedding_models,
    "mAP": hard_ap_text2_image_3m_mAP,
    "R@10": hard_ap_text2_image_3m_Recall,
    "MRR": hard_ap_text2_image_3m_MRR,
    "nDCG@10": hard_ap_text2_image_3m_ndcg
}
hard_ap_text2_image_3m_df = pd.DataFrame(hard_ap_text2_image_3m_data)
hard_ap_text2_image_3m_df = pd.DataFrame(hard_ap_text2_image_3m_df).sort_values(by="mAP", ascending=False).reset_index(drop=True)


# Gradio interface with sortable Dataframe
with gr.Blocks(css="""
    .gradio-container {
        display: flex;
        justify-content: center;
        align-items: center;
        min-height: 100vh;
        flex-direction: column;
    }
""") as demo:
    gr.Markdown("# Ecommerce Embedding Model Benchmarks")
    
    gr.Markdown("This Space contains benchmark results conducted as part of the release of our ecommerce embedding models: [**`Marqo-Ecommerce-L`**](https://huggingface.co/Marqo/marqo-ecommerce-embeddings-L) and [**`Marqo-Ecommerce-B`**](https://huggingface.co/Marqo/marqo-ecommerce-embeddings-B). ")
    gr.Markdown("Our benchmarking process was divided into two distinct regimes, each using different datasets of ecommerce product listings: **marqo-ecommerce-hard** and **marqo-ecommerce-easy**. Both datasets contained product images and text and only differed in size. The \"easy\" dataset is approximately 10-30 times smaller (200k vs 4M products), and designed to accommodate rate-limited models, specifically Cohere-Embeddings-v3 and GCP-Vertex (with limits of 0.66 rps and 2 rps respectively). The \"hard\" dataset represents the true challenge, since it contains four million ecommerce product listings and is more representative of real-world ecommerce search scenarios.")
    gr.Markdown('Within both these scenarios, the models were benchmarked against three different tasks:')
    gr.Markdown('- **Google Shopping Text-to-Image**')
    gr.Markdown('- **Google Shopping Category-to-Image**')
    gr.Markdown('- **Amazon Products Text-to-Image**')
    gr.Markdown('As part of this launch, we also released two evaluation datasets: [`Marqo/google-shopping-general-eval`](https://huggingface.co/datasets/Marqo/google-shopping-general-eval) and [`Marqo/amazon-products-eval`](https://huggingface.co/datasets/Marqo/amazon-products-eval).')
    gr.Markdown('For more information on these models, benchmark results, and how you can run these evaluations yourself, visit our [blog post](https://www.marqo.ai/blog/introducing-marqos-ecommerce-embedding-models).')

    # Hard
    gr.Markdown('## Marqo-Ecommerce-Hard')
    gr.Markdown('### Google Shopping Text to Image 1m')
    gr.Dataframe(value=hard_gs_text2_image_1m_df, headers="keys", interactive=True, datatype=["html", "number", "number", "number", "number"])

    gr.Markdown('### Google Shopping Category to Image 1m')
    gr.Dataframe(value=hard_gs_category2_image_1m_df, headers="keys", interactive=True, datatype=["html", "number", "number", "number", "number"])

    gr.Markdown('###  Amazon Products Text to Image 3m')
    gr.Dataframe(value=hard_ap_text2_image_3m_df, headers="keys", interactive=True, datatype=["html", "number", "number", "number", "number"])
    
    # Easy
    gr.Markdown('## Marqo-Ecommerce-Easy')
    gr.Markdown('### Google Shopping Text to Image')
    gr.Dataframe(value=gs_text2_image_1m_df, headers="keys", interactive=True, datatype=["html", "number", "number", "number", "number"])

    gr.Markdown('### Google Shopping Category to Image')
    gr.Dataframe(value=gs_category2_image_1m_df, headers="keys", interactive=True, datatype=["html", "number", "number", "number", "number"])

    gr.Markdown('### Amazon Products Text to Image')
    gr.Dataframe(value=ap_text2_image_3m_df, headers="keys", interactive=True, datatype=["html", "number", "number", "number", "number"])

demo.launch()