Functional

.gitignore

@@ -1,4 +1,4 @@
 Environment/
 out/
 flagged/
-
+Evaluations/

app.py

@@ -2,6 +2,9 @@ import gradio as gr
 import subprocess
 import os
 import sys
+import time
+import pandas as pd
+from threading import Thread
 
 # Add the path to the "src" directory of detect-pretrain-code-contamination to the sys.path
 project_root = os.path.abspath(os.path.join(os.path.dirname(__file__), "detect-pretrain-code-contamination"))
@@ -9,17 +12,300 @@ src_dir = os.path.join(project_root, "src")
 sys.path.insert(0, src_dir)
 
 import run as evaluator  # Import the run module
+from src.css_html import custom_css
+from src.text_content import ABOUT_TEXT, SUBMISSION_TEXT, SUBMISSION_TEXT_2
+from src.envs import API, H4_TOKEN, REPO_ID
+from huggingface_hub import HfApi
+from src.utils import (
+    AutoEvalColumn,
+    fields,
+    is_model_on_hub,
+    make_clickable_names,
+    styled_error,
+    styled_message,
+)
 
-def evaluate(model):
+COLS = [c.name for c in fields(AutoEvalColumn) if not c.hidden]
+TYPES = [c.type for c in fields(AutoEvalColumn) if not c.hidden]
+COLS_LITE = [c.name for c in fields(AutoEvalColumn) if c.displayed_by_default and not c.hidden]
+TYPES_LITE = [c.type for c in fields(AutoEvalColumn) if c.displayed_by_default and not c.hidden]
+
+# CONFIGURATION:
+ref_model = "huggyllama/llama-7b"
+test_datasets = ["truthful_qa","cais/mmlu","ai2_arc","gsm8k","Rowan/hellaswag","winogrande"]
+modelQueue = []
+
+def restart_space(): #Most dumbest update function to ever exist, I'm sobbing in tears as I've tried to make gradio update the leaderboard literally any other way.
+    API.restart_space(repo_id=REPO_ID, token=H4_TOKEN)
+
+
+def save_to_txt(model, results, model_type):
+    file_path = "data/code_eval_board.csv"
+
+    with open(file_path, "a") as f:
+        f.write(f"\n{model_type},{model}," + str(results["arc"]) + "," + str(results["hellaswag"]) + "," + str(results["mmlu"]) + "," + str(results["truthfulQA"]) + "," + str(results["winogrande"]) + "," + str(results["gsm8k"]))
+        f.close()
+
+    restart_space()
+
+def run_test(model,ref_model,data):
+    print(f"|| TESTING {data} ||")
     return evaluator.main(
-                target_model="roneneldan/TinyStories-1M",
-                ref_model="roneneldan/TinyStories-Instruct-1M",
+                target_model=f"{model}",
+                ref_model=f"{ref_model}",
                 output_dir="out",
-                data="truthful_qa",
+                data=f"{data}",
                 length=64,
                 key_name="input",
                 ratio_gen=0.4
-            )  # Call the run_main function directly
+            ) # Call the main function in detect-pretrain-code-contamination/src/run.py
+
+def evaluate(model,model_type):
+    global ref_model
+    print(f"|| EVALUATING {model} ||")
+    results = {
+        "arc": run_test(model, ref_model, test_datasets[2]),
+        "hellaswag": run_test(model, ref_model, test_datasets[4]),
+        "mmlu": run_test(model, ref_model, test_datasets[1]),
+        "truthfulQA": run_test(model, ref_model, test_datasets[0]),
+        "winogrande": run_test(model, ref_model, test_datasets[5]),
+        "gsm8k": run_test(model, ref_model, test_datasets[3]),
+        "ref_model": ref_model,
+    }
+
+    # Save to .txt file in /Evaluations/{model}
+    save_to_txt(model, results, model_type)
+    return "\n".join([f"{k}:{results[k]}" for k in results])
+
+def worker_thread():
+    global modelQueue, server
+    while True:
+        for submission in modelQueue:
+            evaluate(submission[0],submission[1].split(" ")[0])
+            modelQueue.pop(modelQueue.index(submission))
+            time.sleep(1)
+        time.sleep(1)
+
+def queue(model,model_type):
+    global modelQueue
+    modelQueue.append([model,model_type])
+    print(f"QUEUE:\n{modelQueue}")
+
+
+### bigcode/bigcode-models-leaderboard
+def add_new_eval(
+    model: str,
+    revision: str,
+    precision: str,
+    model_type: str,
+):
+    precision = precision
+
+    if model_type is None or model_type == "" or model_type == []:
+        return styled_error("Please select a model type.")
+    print(model_type)
+    # check the model actually exists before adding the eval
+    if revision == "":
+        revision = "main"
+
+    model_on_hub, error = is_model_on_hub(model, revision)
+    if not model_on_hub:
+        return styled_error(f'Model "{model}" {error}')
+
+    print("Adding new eval")
+    queue(model,model_type)
+    return styled_message("Your request has been submitted to the evaluation queue!\n")
+
+def select_columns(df, columns):
+    always_here_cols = [
+        AutoEvalColumn.model_type_symbol.name,
+        AutoEvalColumn.model.name,
+    ]
+    # We use COLS to maintain sorting
+    filtered_df = df[
+        always_here_cols + [c for c in COLS if c in df.columns and c in columns]
+    ]
+    return filtered_df
+
+
+def filter_items(df, leaderboard_table, query):
+    if query == "All":
+        return df[leaderboard_table.columns]
+    else:
+        query = query[0]  # take only the emoji character
+    filtered_df = df[(df["T"] == query)]
+    return filtered_df[leaderboard_table.columns]
+
+def search_table(df, leaderboard_table, query):
+    filtered_df = df[(df["Models"].str.contains(query, case=False))]
+    return filtered_df[leaderboard_table.columns]
+
+demo = gr.Blocks(css=custom_css)
+with demo:
+    with gr.Row():
+        gr.Markdown(
+            """<div style="text-align: center;"><h1> 📄 LLM Contamination Detector </h1></div>\
+            <br>\
+            <p>Inspired from the <a href="https://huggingface.co/spaces/HuggingFaceH4/open_llm_leaderboard">🤗 Open LLM Leaderboard</a> and <a href="https://huggingface.co/spaces/bigcode/bigcode-models-leaderboard">🤗 Big Code Models Leaderboard ⭐</a>, we use an implementation of <a href="https://huggingface.co/papers/2310.16789">Detecting Pretraining Data from Large Language Models</a> paper found in <a href="https://github.com/swj0419/detect-pretrain-code-contamination/tree/master">this github repo</a>, to provide contamination scores for LLMs on the datasets used by Open LLM Leaderboard.\
+            This space should NOT be used to flag or accuse models of cheating / being contamined, instead, it should form part of a holistic assesment by the parties involved.</p>""",
+            elem_classes="markdown-text",
+        )
+
+    with gr.Tabs(elem_classes="tab-buttons") as tabs:
+        with gr.Column():
+            with gr.Tabs(elem_classes="A100-tabs") as A100_tabs:
+                with gr.TabItem("🔍 Evaluations", id=0):
+                    with gr.Column():
+                        with gr.Accordion("➡️ See filters", open=False):
+                            shown_columns = gr.CheckboxGroup(
+                                choices=[
+                                    c
+                                    for c in COLS
+                                    if c
+                                    not in [
+                                        AutoEvalColumn.dummy.name,
+                                        AutoEvalColumn.model.name,
+                                        AutoEvalColumn.model_type_symbol.name,
+                                    ]
+                                ],
+                                value=[
+                                    c
+                                    for c in COLS_LITE
+                                    if c
+                                    not in [
+                                        AutoEvalColumn.dummy.name,
+                                        AutoEvalColumn.model.name,
+                                        AutoEvalColumn.model_type_symbol.name,
+                                    ]
+                                ],
+                                label="",
+                                elem_id="column-select",
+                                interactive=True,
+                            )
+                        # with gr.Column(min_width=780):
+                        with gr.Row():
+                            search_bar = gr.Textbox(
+                                placeholder="🔍 Search for a model and press ENTER...",
+                                show_label=False,
+                                elem_id="search-bar",
+                            )
+                            filter_columns = gr.Radio(
+                                label="⏚ Filter model types",
+                                choices=["All", "🟢 Base", "🔶 Finetuned"],
+                                value="All",
+                                elem_id="filter-columns",
+                            )
+
+                    df = pd.read_csv("data/code_eval_board.csv")
+                    leaderboard_df = gr.components.Dataframe(
+                        value=df[
+                            [
+                                AutoEvalColumn.model_type_symbol.name,
+                                AutoEvalColumn.model.name,
+                            ]
+                            + shown_columns.value
+                        ],
+                        headers=[
+                            AutoEvalColumn.model_type_symbol.name,
+                            AutoEvalColumn.model.name,
+                        ]
+                        + shown_columns.value,
+                        datatype=TYPES,
+                        elem_id="leaderboard-table",
+                        interactive=False,
+                    )
+
+                    hidden_leaderboard_df = gr.components.Dataframe(
+                        value=df,
+                        headers=COLS,
+                        datatype=["str" for _ in range(len(COLS))],
+                        visible=False,
+                    )
+
+                    search_bar.submit(
+                        search_table,
+                        [hidden_leaderboard_df, leaderboard_df, search_bar],
+                        leaderboard_df,
+                    )
+
+                    filter_columns.change(
+                        filter_items,
+                        [hidden_leaderboard_df, leaderboard_df, filter_columns],
+                        leaderboard_df,
+                    )
+
+                    shown_columns.change(
+                        select_columns,
+                        [hidden_leaderboard_df, shown_columns],
+                        leaderboard_df,
+                    )
+
+                    gr.Markdown(
+                        """
+                    **Notes:**
+                    - The Huggingface team is working on their own implementation of this paper as a space, I'll be leaving this space up until that's available.
+                    - Some scores may not be entirely accurate according to the paper cited as I still work out the kinks and innacuracies of this implementation.
+                    - For any issues, questions, or comments either open a discussion in this space's community tab or message me directly to my discord: yeyito777.
+                    - Make sure to check the pinned discussion in this space's community tab for implementation details I'm not 100% about.
+                    """,
+                        elem_classes="markdown-text",
+                    )
+
+                with gr.TabItem("📝 About", id=2):
+                    gr.Markdown(ABOUT_TEXT, elem_classes="markdown-text")
+                with gr.TabItem("🛠️ Submit models", id=3):
+                    gr.Markdown(SUBMISSION_TEXT)
+                    gr.Markdown(
+                        "## 📤  Submit a model here:", elem_classes="markdown-text"
+                    )
+                    with gr.Column():
+                        with gr.Row():
+                            model_name = gr.Textbox(label="Model name")
+                            revision_name = gr.Textbox(
+                                label="revision", placeholder="main"
+                            )
+                        with gr.Row():
+                            precision = gr.Dropdown(
+                                choices=[
+                                    "float16",
+                                    "bfloat16",
+                                    "8bit",
+                                    "4bit",
+                                ],
+                                label="Precision",
+                                multiselect=False,
+                                value="float16",
+                                interactive=True,
+                            )
+                            model_type = gr.Dropdown(
+                                choices=["🟢 base", "🔶 instruction-tuned"],
+                                label="Model type",
+                                multiselect=False,
+                                value=None,
+                                interactive=True,
+                            )
+                        submit_button = gr.Button("Submit Eval")
+                        submission_result = gr.Markdown()
+                        submit_button.click(
+                            add_new_eval,
+                            inputs=[model_name, revision_name, precision, model_type],
+                            outputs=[submission_result],
+                        )
+                        gr.Markdown(SUBMISSION_TEXT_2)
+
+thread = Thread(target=worker_thread)
+thread.start()
+demo.launch()
+
+# Some worries:
+# 1. Am I testing things correctly in eval.py, following the template format?
+
+# 2. Am I choosing the correct splits in run.py? The higherarchy I use is: test > val > train
+#   (As in: if test exists, I go with that, then validation, then default)
+
+# 3. I decided to go with winogrande_debiased instead of winogrande_l arbitrarily.
+#   (Not sure which one open llm leaderboard uses, or what is the standard)
+
+# 4. I'm unsure why in eval.py we append the output at the end of the input.
 
-iface = gr.Interface(fn=evaluate, inputs="text", outputs="text")
-iface.launch()
+# 5. Currently I'm using huggyllama/llama-7b as ref_model, should I switch to llama2-7B? Maybe Mistral-7B?

data/code_eval_board.csv

@@ -0,0 +1,2 @@
+T,Models,ARC,HellaSwag,MMLU,TruthfulQA,Winogrande,GSM8K
+🟢,roneneldan/TinyStories-1M,0.1 % 0.1,0.1 % 0.1,0.1 % 0.1,0.1 % 0.1,0.1 % 0.1,0.1 % 0.1

requirements.txt

@@ -8,3 +8,4 @@ matplotlib
 scikit-learn
 accelerate
 gradio
+plotly

src/add_json_csv.py

@@ -0,0 +1,108 @@
+
+import pandas as pd
+from utils import model_hyperlink
+
+def add_model_readme(df):
+    # write model ids to README.md
+    with open("README.md", "r") as f:
+        lines = f.readlines()
+    
+    links = df["Links"].astype(str)
+    for link in links:
+        try:
+            model_id = link.split(".co/")[1]
+            # verify line doesn't exist
+            if f"- {model_id}\n" in lines:
+                continue
+            lines.insert(-1, f"- {model_id}\n")
+        except IndexError:
+            print(f"link {link} is not valid")
+
+    with open("README.md", "w") as f:
+        f.writelines(lines)
+
+df = pd.read_csv("data/raw_scores.csv")
+COLS = df.columns.to_list()
+# add column models_query with same values a smodels at the end of columns
+df.insert(len(COLS), "models_query", df["Models"])
+print(f"all cols {df.columns.to_list()}")
+# average score
+mean_columns = df.iloc[:,5:-3]
+# print cols in mean_columns
+print("cols", mean_columns.columns.to_list())
+df.insert(len(mean_columns.columns.to_list()), "Average score", mean_columns.mean(axis=1).round(2))
+
+# add win rate columns for each language
+old_size = len(df.columns)
+
+for col in df.columns[6:-2]:
+    df[col + " rank"] = df[col].rank(ascending=False)
+    df[col + " rank"] = len(df) - (df[col + " rank"] - 1)
+df["Win Rate"] = df.iloc[:, old_size:].mean(axis=1).round(2)
+df = df.drop(df.columns[old_size:-1], axis=1)
+df = df[["Models", "Size (B)", "Win Rate"] + df.columns[2:-1].tolist()]
+
+# sort with regard to column win rate
+df = df.sort_values(by=["Win Rate"], ascending=False)
+# add column with model links as https://huggingface.co/WizardLM/WizardCoder-15B-V1.0, https://huggingface.co/bigcode/starcoder, https://huggingface.co/bigcode/starcoderbase, https://huggingface.co/bigcode/starcoderbase-7b,
+# https://huggingface.co/bigcode/starcoderbase-3b, https://huggingface.co/bigcode/starcoderbase-1b, https://huggingface.co/bigcode/santacoder, https://huggingface.co/replit/replit-code-v1-3b, https://huggingface.co/THUDM/codegeex2-6b
+
+links = {
+    "WizardCoder-15B-V1.0": "https://huggingface.co/WizardLM/WizardCoder-15B-V1.0",
+    "WizardCoder-3B-V1.0": "https://huggingface.co/WizardLM/WizardCoder-3B-V1.0",
+    "WizardCoder-1B-V1.0": "https://huggingface.co/WizardLM/WizardCoder-1B-V1.0",
+    "WizardCoder-Python-34B-V1.0": "https://huggingface.co/WizardLM/WizardCoder-Python-34B-V1.0",
+    "WizardCoder-Python-13B-V1.0": "https://huggingface.co/WizardLM/WizardCoder-Python-13B-V1.0",
+    "OctoCoder-15B": "https://huggingface.co/bigcode/octocoder",
+    "OctoGeeX-7B": "https://huggingface.co/bigcode/octogeex",
+    "StableCode-3B": "https://huggingface.co/stabilityai/stablecode-completion-alpha-3b",
+    "StarCoder-15B": "https://huggingface.co/bigcode/starcoder",
+    "StarCoderBase-15B": "https://huggingface.co/bigcode/starcoderbase",
+    "StarCoderBase-7B": "https://huggingface.co/bigcode/starcoderbase-7b",
+    "StarCoderBase-3B": "https://huggingface.co/bigcode/starcoderbase-3b",
+    "StarCoderBase-1.1B": "https://huggingface.co/bigcode/starcoderbase-1b",
+    "SantaCoder-1.1B": "https://huggingface.co/bigcode/santacoder",
+    "Replit-2.7B": "https://huggingface.co/replit/replit-code-v1-3b",
+    "CodeGeex2-6B": "https://huggingface.co/THUDM/codegeex2-6b",
+    "CodeGen25-7B-multi": "https://huggingface.co/Salesforce/codegen25-7b-multi",
+    "CodeGen25-7B-mono": "https://huggingface.co/Salesforce/codegen25-7b-mono",
+    "CodeGen-16B-Multi": "https://huggingface.co/Salesforce/codegen-16B-multi",
+    "DeciCoder-1B": "https://huggingface.co/Deci/DeciCoder-1b",
+    "Phind-CodeLlama-34B-v1": "https://huggingface.co/phind/Phind-CodeLlama-34B-v1",
+    "Phind-CodeLlama-34B-Python-v1": "https://huggingface.co/phind/Phind-CodeLlama-34B-Python-v1",
+    "Phind-CodeLlama-34B-v2": "https://huggingface.co/phind/Phind-CodeLlama-34B-v2",
+    "Falcon-180B": "https://huggingface.co/tiiuae/falcon-180B",
+    "Refact-1.6B": "https://huggingface.co/smallcloudai/Refact-1_6B-fim",
+    "Phi-1": "https://huggingface.co/microsoft/phi-1",
+    "CodeShell-7B": "https://huggingface.co/WisdomShell/CodeShell-7B",
+    "DeepSeek-Coder-1b-base": "https://huggingface.co/deepseek-ai/deepseek-coder-1.3b-base",
+    "DeepSeek-Coder-7b-base": "https://huggingface.co/deepseek-ai/deepseek-coder-6.7b-base",
+    "DeepSeek-Coder-33b-base": "https://huggingface.co/deepseek-ai/deepseek-coder-33b-base",
+}
+
+codellamas = ['CodeLlama-7b', 'CodeLlama-7b-Python', 'CodeLlama-7b-Instruct', 'CodeLlama-13b', 'CodeLlama-13b-Python', 'CodeLlama-13b-Instruct', 'CodeLlama-34b', 'CodeLlama-34b-Python', 'CodeLlama-34b-Instruct']
+for codellama in codellamas:
+    links[codellama] = f"https://huggingface.co/codellama/{codellama}-hf"
+
+df["Links"] = df["Models"].map(links)
+
+df.insert(0, "T", "🟢")
+patterns = ["WizardCoder", "Octo", "Instruct", "Phind", "Refact"]
+df.loc[df["Models"].str.contains('|'.join(patterns)), "T"] = "🔶"
+df.loc[df["Models"].str.contains('|'.join(patterns)), "T"] = "🔶"
+df.loc[df["Models"].str.contains('|'.join(["CodeShell", "DeepSeek"])), "T"] = "🔴"
+# add clumn submission_pr with empty fiels except for CodeShell with link AA
+df["Submission PR"] = ""
+df.loc[df["Models"].str.contains('|'.join(["CodeShell"])), "Submission PR"] = "https://huggingface.co/spaces/bigcode/bigcode-models-leaderboard/discussions/16"
+df.loc[df["Models"].str.contains('|'.join(["DeepSeek-Coder-1b-base"])), "Submission PR"] = "https://huggingface.co/spaces/bigcode/bigcode-models-leaderboard/discussions/33"
+df.loc[df["Models"].str.contains('|'.join(["DeepSeek-Coder-7b-base"])), "Submission PR"] = "https://huggingface.co/spaces/bigcode/bigcode-models-leaderboard/discussions/32"
+df.loc[df["Models"].str.contains('|'.join(["DeepSeek-Coder-33b-base"])), "Submission PR"] = "https://huggingface.co/spaces/bigcode/bigcode-models-leaderboard/discussions/31"
+
+
+# print first 5 rows and 10 cols
+print(df.iloc[:5, :-1])
+df.to_csv("data/code_eval_board.csv", index=False)
+
+# fill readme
+add_model_readme(df)
+print("Readme filled")

src/build.py

@@ -0,0 +1,107 @@
+import pandas as pd
+from utils import model_hyperlink
+
+def add_model_readme(df):
+    # write model ids to README.md
+    with open("README.md", "r") as f:
+        lines = f.readlines()
+    
+    links = df["Links"].astype(str)
+    for link in links:
+        try:
+            model_id = link.split(".co/")[1]
+            # verify line doesn't exist
+            if f"- {model_id}\n" in lines:
+                continue
+            lines.insert(-1, f"- {model_id}\n")
+        except IndexError:
+            print(f"link {link} is not valid")
+
+    with open("README.md", "w") as f:
+        f.writelines(lines)
+
+df = pd.read_csv("data/raw_scores.csv")
+COLS = df.columns.to_list()
+# add column models_query with same values a smodels at the end of columns
+df.insert(len(COLS), "models_query", df["Models"])
+print(f"all cols {df.columns.to_list()}")
+# average score
+mean_columns = df.iloc[:,5:-3]
+# print cols in mean_columns
+print("cols", mean_columns.columns.to_list())
+df.insert(len(mean_columns.columns.to_list()), "Average score", mean_columns.mean(axis=1).round(2))
+
+# add win rate columns for each language
+old_size = len(df.columns)
+
+for col in df.columns[6:-2]:
+    df[col + " rank"] = df[col].rank(ascending=False)
+    df[col + " rank"] = len(df) - (df[col + " rank"] - 1)
+df["Win Rate"] = df.iloc[:, old_size:].mean(axis=1).round(2)
+df = df.drop(df.columns[old_size:-1], axis=1)
+df = df[["Models", "Size (B)", "Win Rate"] + df.columns[2:-1].tolist()]
+
+# sort with regard to column win rate
+df = df.sort_values(by=["Win Rate"], ascending=False)
+# add column with model links as https://huggingface.co/WizardLM/WizardCoder-15B-V1.0, https://huggingface.co/bigcode/starcoder, https://huggingface.co/bigcode/starcoderbase, https://huggingface.co/bigcode/starcoderbase-7b,
+# https://huggingface.co/bigcode/starcoderbase-3b, https://huggingface.co/bigcode/starcoderbase-1b, https://huggingface.co/bigcode/santacoder, https://huggingface.co/replit/replit-code-v1-3b, https://huggingface.co/THUDM/codegeex2-6b
+
+links = {
+    "WizardCoder-15B-V1.0": "https://huggingface.co/WizardLM/WizardCoder-15B-V1.0",
+    "WizardCoder-3B-V1.0": "https://huggingface.co/WizardLM/WizardCoder-3B-V1.0",
+    "WizardCoder-1B-V1.0": "https://huggingface.co/WizardLM/WizardCoder-1B-V1.0",
+    "WizardCoder-Python-34B-V1.0": "https://huggingface.co/WizardLM/WizardCoder-Python-34B-V1.0",
+    "WizardCoder-Python-13B-V1.0": "https://huggingface.co/WizardLM/WizardCoder-Python-13B-V1.0",
+    "OctoCoder-15B": "https://huggingface.co/bigcode/octocoder",
+    "OctoGeeX-7B": "https://huggingface.co/bigcode/octogeex",
+    "StableCode-3B": "https://huggingface.co/stabilityai/stablecode-completion-alpha-3b",
+    "StarCoder-15B": "https://huggingface.co/bigcode/starcoder",
+    "StarCoderBase-15B": "https://huggingface.co/bigcode/starcoderbase",
+    "StarCoderBase-7B": "https://huggingface.co/bigcode/starcoderbase-7b",
+    "StarCoderBase-3B": "https://huggingface.co/bigcode/starcoderbase-3b",
+    "StarCoderBase-1.1B": "https://huggingface.co/bigcode/starcoderbase-1b",
+    "SantaCoder-1.1B": "https://huggingface.co/bigcode/santacoder",
+    "Replit-2.7B": "https://huggingface.co/replit/replit-code-v1-3b",
+    "CodeGeex2-6B": "https://huggingface.co/THUDM/codegeex2-6b",
+    "CodeGen25-7B-multi": "https://huggingface.co/Salesforce/codegen25-7b-multi",
+    "CodeGen25-7B-mono": "https://huggingface.co/Salesforce/codegen25-7b-mono",
+    "CodeGen-16B-Multi": "https://huggingface.co/Salesforce/codegen-16B-multi",
+    "DeciCoder-1B": "https://huggingface.co/Deci/DeciCoder-1b",
+    "Phind-CodeLlama-34B-v1": "https://huggingface.co/phind/Phind-CodeLlama-34B-v1",
+    "Phind-CodeLlama-34B-Python-v1": "https://huggingface.co/phind/Phind-CodeLlama-34B-Python-v1",
+    "Phind-CodeLlama-34B-v2": "https://huggingface.co/phind/Phind-CodeLlama-34B-v2",
+    "Falcon-180B": "https://huggingface.co/tiiuae/falcon-180B",
+    "Refact-1.6B": "https://huggingface.co/smallcloudai/Refact-1_6B-fim",
+    "Phi-1": "https://huggingface.co/microsoft/phi-1",
+    "CodeShell-7B": "https://huggingface.co/WisdomShell/CodeShell-7B",
+    "DeepSeek-Coder-1b-base": "https://huggingface.co/deepseek-ai/deepseek-coder-1.3b-base",
+    "DeepSeek-Coder-7b-base": "https://huggingface.co/deepseek-ai/deepseek-coder-6.7b-base",
+    "DeepSeek-Coder-33b-base": "https://huggingface.co/deepseek-ai/deepseek-coder-33b-base",
+}
+
+codellamas = ['CodeLlama-7b', 'CodeLlama-7b-Python', 'CodeLlama-7b-Instruct', 'CodeLlama-13b', 'CodeLlama-13b-Python', 'CodeLlama-13b-Instruct', 'CodeLlama-34b', 'CodeLlama-34b-Python', 'CodeLlama-34b-Instruct']
+for codellama in codellamas:
+    links[codellama] = f"https://huggingface.co/codellama/{codellama}-hf"
+
+df["Links"] = df["Models"].map(links)
+
+df.insert(0, "T", "🟢")
+patterns = ["WizardCoder", "Octo", "Instruct", "Phind", "Refact"]
+df.loc[df["Models"].str.contains('|'.join(patterns)), "T"] = "🔶"
+df.loc[df["Models"].str.contains('|'.join(patterns)), "T"] = "🔶"
+df.loc[df["Models"].str.contains('|'.join(["CodeShell", "DeepSeek"])), "T"] = "🔴"
+# add clumn submission_pr with empty fiels except for CodeShell with link AA
+df["Submission PR"] = ""
+df.loc[df["Models"].str.contains('|'.join(["CodeShell"])), "Submission PR"] = "https://huggingface.co/spaces/bigcode/bigcode-models-leaderboard/discussions/16"
+df.loc[df["Models"].str.contains('|'.join(["DeepSeek-Coder-1b-base"])), "Submission PR"] = "https://huggingface.co/spaces/bigcode/bigcode-models-leaderboard/discussions/33"
+df.loc[df["Models"].str.contains('|'.join(["DeepSeek-Coder-7b-base"])), "Submission PR"] = "https://huggingface.co/spaces/bigcode/bigcode-models-leaderboard/discussions/32"
+df.loc[df["Models"].str.contains('|'.join(["DeepSeek-Coder-33b-base"])), "Submission PR"] = "https://huggingface.co/spaces/bigcode/bigcode-models-leaderboard/discussions/31"
+
+
+# print first 5 rows and 10 cols
+print(df.iloc[:5, :-1])
+df.to_csv("data/code_eval_board.csv", index=False)
+
+# fill readme
+add_model_readme(df)
+print("Readme filled")

src/css_html.py

@@ -0,0 +1,66 @@
+
+# source: https://huggingface.co/spaces/HuggingFaceH4/open_llm_leaderboard/blob/main/src/assets/css_html_js.py
+custom_css = """
+#changelog-text {
+    font-size: 16px !important;
+}
+#changelog-text h2 {
+    font-size: 18px !important;
+}
+.markdown-text {
+    font-size: 16px !important;
+}
+#models-to-add-text {
+    font-size: 18px !important;
+}
+#citation-button span {
+    font-size: 16px !important;
+}
+#citation-button textarea {
+    font-size: 16px !important;
+}
+#citation-button > label > button {
+    margin: 6px;
+    transform: scale(1.3);
+}
+#leaderboard-table {
+    margin-top: 15px
+}
+#leaderboard-table-lite {
+    margin-top: 15px
+}
+#search-bar-table-box > div:first-child {
+    background: none;
+    border: none;
+}
+ 
+#search-bar {
+    padding: 0px;
+}
+/* Hides the final AutoEvalColumn */
+#llm-benchmark-tab-table table td:last-child,
+#llm-benchmark-tab-table table th:last-child {
+    display: none;
+}
+/* Limit the width of the first AutoEvalColumn so that names don't expand too much */
+table td:first-child,
+table th:first-child {
+    max-width: 400px;
+    overflow: auto;
+    white-space: nowrap;
+}
+.tab-buttons button {
+    font-size: 20px;
+}
+#scale-logo {
+    border-style: none !important;
+    box-shadow: none;
+    display: block;
+    margin-left: auto;
+    margin-right: auto;
+    max-width: 600px;
+}
+#scale-logo .download {
+    display: none;
+}
+"""

src/envs.py

@@ -0,0 +1,6 @@
+import os
+from huggingface_hub import HfApi
+
+H4_TOKEN = os.environ.get("H4_TOKEN", None)
+REPO_ID = "Yeyito/llm_contamination_detector"
+API = HfApi(token=H4_TOKEN)

src/text_content.py

@@ -0,0 +1,20 @@
+
+ABOUT_TEXT = """# Background
+Model contamination is an obstacle that many model creators face and has become a growing issue amongst the top scorers in [🤗 Open LLM Leaderboard](https://huggingface.co/spaces/HuggingFaceH4/open_llm_leaderboard). This work is an implementation of the [Detecting Pretraining Data from Large Language Models](https://huggingface.co/papers/2310.16789) following the template provided by [this github repo](https://github.com/swj0419/detect-pretrain-code-contamination/tree/master). I'm aware the Hugginface Team is working on their own implementation of this working directly with the authors of the paper mentioned above. Until that's ready I hope this serves as a metric for evaluating model contamination in open source llms.
+
+# Disclaimer
+This space should NOT be used to flag or accuse models of cheating / being contamined. Instead, it should form part of a holistic assesment by the parties involved. The main goal of this space is to provide more transparency as to what the contents of the datasets used to train models are take whatever is shown in the evaluation's tab as a grain of salt and draw your own conclusions from the data.
+
+As a final note, I've outlined my main concerns with this implementation in a pinned discussion under the community tab. Any type of help would be greatly appreciated :)"""
+
+SUBMISSION_TEXT = """
+<h1 align="center">
+Submitting models for evaluation.
+</h1>
+This space is still highly experimental, try to not submit any GGUF, GPTQ or AWQ models, but their raw fp16/bf16 versions. Also try to not submit any model above 13B parameters in size as this space is not equipped with the hardware to handle that.
+
+"""
+
+SUBMISSION_TEXT_2 = """
+If you encounter any issues while submitting please make a community post about it, or message my discord directly: yeyito777!
+"""

src/utils.py

@@ -0,0 +1,79 @@
+
+# source: https://huggingface.co/spaces/HuggingFaceH4/open_llm_leaderboard/blob/main/src/utils_display.py
+from dataclasses import dataclass
+import plotly.graph_objects as go
+from transformers import AutoConfig
+
+# These classes are for user facing column names, to avoid having to change them
+# all around the code when a modif is needed
+@dataclass
+class ColumnContent:
+    name: str
+    type: str
+    displayed_by_default: bool
+    hidden: bool = False
+
+
+def fields(raw_class):
+    return [
+        v for k, v in raw_class.__dict__.items() if k[:2] != "__" and k[-2:] != "__"
+    ]
+
+
+@dataclass(frozen=True)
+class AutoEvalColumn:  # Auto evals column
+    model_type_symbol = ColumnContent("T", "str", True)
+    model = ColumnContent("Models", "markdown", True)
+    ARC = ColumnContent("ARC", "number", True)
+    HellaSwag = ColumnContent("HellaSwag", "number", True)
+    MMLU = ColumnContent("MMLU", "number", True)
+    TruthfulQA = ColumnContent("TruthfulQA", "number", True)
+    Winogrande = ColumnContent("Winogrande", "number", True)
+    GSM8K = ColumnContent("GSM8K", "number", True)
+    dummy = ColumnContent("Models", "str", True)
+
+
+def model_hyperlink(link, model_name):
+    return f'<a target="_blank" href="{link}" style="color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;">{model_name}</a>'
+
+
+def make_clickable_names(df):
+    df["Models"] = df.apply(
+        lambda row: model_hyperlink(row["Links"], row["Models"]), axis=1
+    )
+    return df
+
+def styled_error(error):
+    return f"<p style='color: red; font-size: 20px; text-align: center;'>{error}</p>"
+
+
+def styled_warning(warn):
+    return f"<p style='color: orange; font-size: 20px; text-align: center;'>{warn}</p>"
+
+
+def styled_message(message):
+    return f"<p style='color: green; font-size: 20px; text-align: center;'>{message}</p>"
+
+
+def has_no_nan_values(df, columns):
+    return df[columns].notna().all(axis=1)
+
+
+def has_nan_values(df, columns):
+    return df[columns].isna().any(axis=1)
+
+
+def is_model_on_hub(model_name: str, revision: str) -> bool:
+    try:
+        AutoConfig.from_pretrained(model_name, revision=revision, trust_remote_code=False)
+        return True, None
+
+    except ValueError:
+        return (
+            False,
+            "needs to be launched with `trust_remote_code=True`. For safety reason, we do not allow these models to be automatically submitted to the leaderboard.",
+        )
+
+    except Exception as e:
+        print(f"Could not get the model config from the hub.: {e}")
+        return False, "was not found on hub!"