app.py

import gradio as gr
import subprocess
import os
import sys
import time
import pandas as pd
from threading import Thread
import numpy as np

# Add the path to the "src" directory of detect-pretrain-code-contamination to the sys.path
project_root = os.path.abspath(os.path.join(os.path.dirname(__file__), "detect-pretrain-code-contamination"))
src_dir = os.path.join(project_root, "src")
sys.path.insert(0, src_dir)

import run as evaluator  # Import the run module
from src.css_html import custom_css
from src.text_content import ABOUT_TEXT, SUBMISSION_TEXT, SUBMISSION_TEXT_2
from src.envs import API, H4_TOKEN, REPO_ID
from huggingface_hub import HfApi
from src.utils import (
    AutoEvalColumn,
    fields,
    is_model_on_hub,
    make_clickable_names,
    styled_error,
    styled_message,
    EVAL_COLS,
    EVAL_TYPES
)

COLS = [c.name for c in fields(AutoEvalColumn) if not c.hidden]
TYPES = [c.type for c in fields(AutoEvalColumn) if not c.hidden]
COLS_LITE = [c.name for c in fields(AutoEvalColumn) if c.displayed_by_default and not c.hidden]
TYPES_LITE = [c.type for c in fields(AutoEvalColumn) if c.displayed_by_default and not c.hidden]

# CONFIGURATION:
test_datasets = ["truthful_qa","cais/mmlu","ai2_arc","gsm8k","Rowan/hellaswag","winogrande"]
modelQueue = (pd.read_csv('data/queue.csv')).values.tolist()
print(modelQueue)

def restart_space(): #Most dumbest update function to ever exist, I'm sobbing in tears as I've tried to make gradio update the leaderboard literally any other way.
    API.restart_space(repo_id=REPO_ID, token=H4_TOKEN)

def formatr(result):
    result = str(result)
    result = result.split(",")[2].replace(")","")
    result = result.replace(" ","")
    return result

def save_to_txt(model, results, model_type,ref_model):
    file_path = "data/code_eval_board.csv"

    with open(file_path, "a") as f:
        f.write(f"\n{model_type},{model}," + str(formatr(results["arc"])) + "," + str(formatr(results["hellaswag"])) + "," + str(formatr(results["mmlu"])) + "," + str(formatr(results["truthfulQA"])) + "," + str(formatr(results["winogrande"])) + "," + str(formatr(results["gsm8k"])) + f",{ref_model}")
        
        print(f"Finished evaluation of model: {model} using ref_model: {ref_model}")
        print(f"\n{model_type},{model}," + str(formatr(results["arc"])) + "," + str(formatr(results["hellaswag"])) + "," + str(formatr(results["mmlu"])) + "," + str(formatr(results["truthfulQA"])) + "," + str(formatr(results["winogrande"])) + "," + str(formatr(results["gsm8k"])) + f",{ref_model}")
        f.close()

def run_test(model,ref_model,data):
    print(f"|| TESTING {data} ||")
    return evaluator.main(
                target_model=f"{model}",
                ref_model=f"{ref_model}",
                output_dir="out",
                data=f"{data}",
                length=64,
                key_name="input",
                ratio_gen=0.4
            ) # Call the main function in detect-pretrain-code-contamination/src/run.py

def evaluate(model,model_type,ref_model):
    print(f"|| EVALUATING {model} ||")
    results = {
        "arc": run_test(model, ref_model, test_datasets[2]),
        "hellaswag": run_test(model, ref_model, test_datasets[4]),
        "mmlu": run_test(model, ref_model, test_datasets[1]),
        "truthfulQA": run_test(model, ref_model, test_datasets[0]),
        "winogrande": run_test(model, ref_model, test_datasets[5]),
        "gsm8k": run_test(model, ref_model, test_datasets[3]),
        "ref_model": ref_model,
    }

    # Save to .txt file in /Evaluations/{model}
    save_to_txt(model, results, model_type,ref_model)
    return "\n".join([f"{k}:{results[k]}" for k in results])

def worker_thread():
    global modelQueue, server
    while True:
        for submission in modelQueue:
            #evaluate(submission[1],submission[0].split(" ")[0],submission[2])
            #modelQueue.pop(modelQueue.index(submission))
            #exit()
            
            #The exit above is temporal while I figure out how to unload a model from a thread or similar.
            # Uncomment those lines in order to begin testing, I test these models outside of this space and later commit the results back.
            # I highly encourage you to try to reproduce the results I get using your own implementation.
            # Do NOT take anything listed here as fact, as I'm not 100% my implementation works as intended.
            # Take whatever you see in the leaderboard as a grain of salt, do NOT accuse models of cheating just because of their placement here alone.

            time.sleep(1)
        
        time.sleep(1)

def queue(model,model_type,ref_model):
    global modelQueue
    modelQueue.append([model_type,model,ref_model])

    file_path = "data/queue.csv"
    with open(file_path, "a") as f:
        model = model.strip()
        ref_model = ref_model.strip()
        f.write(f"\n{model_type},{model},{ref_model}")
        f.close()
    print(f"QUEUE:\n{modelQueue}")

### bigcode/bigcode-models-leaderboard
def add_new_eval(
    model: str,
    revision: str,
    ref_model: str,
    model_type: str,
):
    ref_model = ref_model

    if model_type is None or model_type == "" or model_type == []:
        return styled_error("Please select a model type.")
    print(model_type)
    # check the model actually exists before adding the eval
    if revision == "":
        revision = "main"

    model_on_hub, error = is_model_on_hub(model, revision)
    if not model_on_hub:
        return styled_error(f'Model "{model}" {error}')

    print("Adding new eval")
    queue(model,model_type,ref_model)
    return styled_message("Your request has been submitted to the evaluation queue!\n")

def select_columns(df, columns):
    always_here_cols = [
        AutoEvalColumn.model_type_symbol.name,
        AutoEvalColumn.model.name,
    ]
    # We use COLS to maintain sorting
    filtered_df = df[
        always_here_cols + [c for c in COLS if c in df.columns and c in columns]
    ]
    return filtered_df


def filter_items(df, leaderboard_table, query):
    if query == "All":
        return df[leaderboard_table.columns]
    else:
        query = query[0]  # take only the emoji character
    filtered_df = df[(df["T"] == query)]
    return filtered_df[leaderboard_table.columns]

def search_table(df, leaderboard_table, query):
    filtered_df = df[(df["Models"].str.contains(query, case=False))]
    return filtered_df[leaderboard_table.columns]

demo = gr.Blocks(css=custom_css)
with demo:
    with gr.Row():
        gr.Markdown(
            """<div style="text-align: center;"><h1> 📄 LLM Contamination Detector </h1></div>\
            <br>\
            <p>Inspired from the <a href="https://huggingface.co/spaces/HuggingFaceH4/open_llm_leaderboard">🤗 Open LLM Leaderboard</a> and <a href="https://huggingface.co/spaces/bigcode/bigcode-models-leaderboard">🤗 Big Code Models Leaderboard ⭐</a>, we use an implementation of <a href="https://huggingface.co/papers/2310.16789">Detecting Pretraining Data from Large Language Models</a> paper found in <a href="https://github.com/swj0419/detect-pretrain-code-contamination/tree/master">this github repo</a>, to provide contamination scores for LLMs on the datasets used by Open LLM Leaderboard.\
            This space should NOT be used to flag or accuse models of cheating / being contamined, instead, it should form part of a holistic assesment by the parties involved.</p>""",
            elem_classes="markdown-text",
        )

    with gr.Tabs(elem_classes="tab-buttons") as tabs:
        with gr.Column():
            with gr.Tabs(elem_classes="A100-tabs") as A100_tabs:
                with gr.TabItem("🔍 Evaluations", id=0):
                    with gr.Column():
                        with gr.Accordion("➡️ See filters", open=False):
                            shown_columns = gr.CheckboxGroup(
                                choices=[
                                    c
                                    for c in COLS
                                    if c
                                    not in [
                                        AutoEvalColumn.dummy.name,
                                        AutoEvalColumn.model.name,
                                        AutoEvalColumn.model_type_symbol.name,
                                    ]
                                ],
                                value=[
                                    c
                                    for c in COLS_LITE
                                    if c
                                    not in [
                                        AutoEvalColumn.dummy.name,
                                        AutoEvalColumn.model.name,
                                        AutoEvalColumn.model_type_symbol.name,
                                    ]
                                ],
                                label="",
                                elem_id="column-select",
                                interactive=True,
                            )
                        # with gr.Column(min_width=780):
                        with gr.Row():
                            search_bar = gr.Textbox(
                                placeholder="🔍 Search for a model and press ENTER...",
                                show_label=False,
                                elem_id="search-bar",
                            )
                            filter_columns = gr.Radio(
                                label="⏚ Filter model types",
                                choices=["All", "🟢 Base", "🔶 Finetuned"],
                                value="All",
                                elem_id="filter-columns",
                            )

                    df = pd.read_csv("data/code_eval_board.csv")
                    leaderboard_df = gr.components.Dataframe(
                        value=df[
                            [
                                AutoEvalColumn.model_type_symbol.name,
                                AutoEvalColumn.model.name,
                            ]
                            + shown_columns.value
                        ],
                        headers=[
                            AutoEvalColumn.model_type_symbol.name,
                            AutoEvalColumn.model.name,
                        ]
                        + shown_columns.value,
                        datatype=TYPES,
                        elem_id="leaderboard-table",
                        interactive=False,
                    )

                    hidden_leaderboard_df = gr.components.Dataframe(
                        value=df,
                        headers=COLS,
                        datatype=["str" for _ in range(len(COLS))],
                        visible=False,
                    )

                    search_bar.submit(
                        search_table,
                        [hidden_leaderboard_df, leaderboard_df, search_bar],
                        leaderboard_df,
                    )

                    filter_columns.change(
                        filter_items,
                        [hidden_leaderboard_df, leaderboard_df, filter_columns],
                        leaderboard_df,
                    )

                    shown_columns.change(
                        select_columns,
                        [hidden_leaderboard_df, shown_columns],
                        leaderboard_df,
                    )

                    gr.Markdown(
                        """
                    **Notes:**
                    - The Huggingface team is working on their own implementation of this paper as a space, I'll be leaving this space up until that's available.
                    - Some scores may not be entirely accurate according to the paper cited as I still work out the kinks and innacuracies of this implementation.
                    - For any issues, questions, or comments either open a discussion in this space's community tab or message me directly to my discord: yeyito777.
                    - Make sure to check the pinned discussion in this space's community tab for implementation details I'm not 100% about.
                    """,
                        elem_classes="markdown-text",
                    )

                with gr.TabItem("📝 About", id=2):
                    gr.Markdown(ABOUT_TEXT, elem_classes="markdown-text")
                with gr.TabItem("🛠️ Submit models", id=3):
                    gr.Markdown(SUBMISSION_TEXT)
                    gr.Markdown(
                        "## 📤  Submit a model here:", elem_classes="markdown-text"
                    )
                    with gr.Column():
                        with gr.Column():
                            with gr.Accordion(
                                f"⏳ Evaluation Queue ({len(modelQueue)})",
                                open=False,
                            ):
                                with gr.Row():
                                    finished_eval_table = gr.components.Dataframe(
                                            value=pd.DataFrame(modelQueue, columns=['Type','Model','Reference Model']),
                                    )
                        with gr.Row():
                            model_name = gr.Textbox(label="Model name")
                            revision_name = gr.Textbox(
                                label="revision", placeholder="main"
                            )
                        with gr.Row():
                            ref_model = gr.Dropdown(
                                choices=[
                                    "mistralai/Mistral-7B-v0.1",
                                    "huggyllama/llama-7b",
                                    "NousResearch/Llama-2-7b-hf",
                                    "upstage/SOLAR-10.7B-v1.0",
                                ],
                                label="Reference Model",
                                multiselect=False,
                                value="mistralai/Mistral-7B-v0.1",
                                interactive=True,
                            )
                            model_type = gr.Dropdown(
                                choices=["🟢 base", "🔶 finetuned"],
                                label="Model type",
                                multiselect=False,
                                value=None,
                                interactive=True,
                            )
                        submit_button = gr.Button("Submit Eval")
                        submission_result = gr.Markdown()
                        submit_button.click(
                            add_new_eval,
                            inputs=[model_name, revision_name, ref_model, model_type],
                            outputs=[submission_result],
                        )
                        gr.Markdown(SUBMISSION_TEXT_2)

thread = Thread(target=worker_thread)
thread.start()
demo.launch(share=True)

# Some worries:
# 1. Am I testing things correctly in eval.py, following the template format?

# 2. Am I choosing the correct splits in run.py? The higherarchy I use is: test > val > train
#   (As in: if test exists, I go with that, then validation, then default)

# 3. I decided to go with winogrande_debiased instead of winogrande_l arbitrarily.
#   (Not sure which one open llm leaderboard uses, or what is the standard)

# 4. I'm unsure why in eval.py we append the output at the end of the input.

# 5. Currently I'm using huggyllama/llama-7b as ref_model, should I switch to llama2-7B? Maybe Mistral-7B?