first commit, migrated from chaeeun's local

README.md

@@ -1,13 +1,13 @@
 ---
-title: Biomed Probing Leaderboard
-emoji: 🏆
+title: Test Leaderboard
+emoji: 🐢
 colorFrom: pink
-colorTo: yellow
+colorTo: red
 sdk: gradio
-sdk_version: 4.16.0
+sdk_version: 4.15.0
 app_file: app.py
 pinned: false
-license: mit
+license: apache-2.0
 ---
 
 Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference

app.py

@@ -0,0 +1,343 @@
+import gradio as gr
+import pandas as pd
+
+from src.display.css_html_js import custom_css
+
+from src.display.about import (
+    CITATION_BUTTON_LABEL,
+    CITATION_BUTTON_TEXT,
+    EVALUATION_QUEUE_TEXT,
+    INTRODUCTION_TEXT,
+    LLM_BENCHMARKS_TEXT,
+    LLM_BENCHMARKS_DETAILS,
+    FAQ_TEXT,
+    TITLE,
+)
+
+from src.display.utils import (
+    BENCHMARK_COLS,
+    COLS,
+    EVAL_COLS,
+    EVAL_TYPES,
+    NUMERIC_INTERVALS,
+    TYPES,
+    AutoEvalColumn,
+    ModelType,
+    fields,
+    WeightType,
+    Precision
+)
+
+from src.populate import get_evaluation_queue_df, get_leaderboard_df
+
+from src.envs import API, EVAL_REQUESTS_PATH, EVAL_RESULTS_PATH, H4_TOKEN, IS_PUBLIC, QUEUE_REPO, REPO_ID, RESULTS_REPO
+from src.submission.submit import add_new_eval
+
+from src.display.utils import Tasks
+
+from huggingface_hub import snapshot_download
+
+## ------- ## ------- ## ------- ## ------- ## ------- ## ------- ## ------- ## ------- ## ------- ## ------- ## ------- ## ------- 
+
+def restart_space():
+    API.restart_space(repo_id=REPO_ID)#, token=H4_TOKEN)
+
+def ui_snapshot_download(repo_id, local_dir, repo_type, tqdm_class, etag_timeout):
+    try:
+        print(f"local_dir for snapshot download = {local_dir}")
+        snapshot_download(repo_id=repo_id, local_dir=local_dir, repo_type=repo_type, tqdm_class=tqdm_class, etag_timeout=etag_timeout)
+    except Exception:
+        print(f"ui_snapshot_download failed. restarting space...")
+        restart_space()
+
+# Searching and filtering
+def update_table(hidden_df: pd.DataFrame, columns: list, type_query: list, precision_query: list, size_query: list, query: str):
+    print(f"hidden_df = {hidden_df}")
+    show_deleted = True
+    filtered_df = filter_models(hidden_df, type_query, size_query, precision_query, show_deleted)
+
+    print(f"filtered_df = {filtered_df}")
+    filtered_df = filter_queries(query, filtered_df)
+    df = select_columns(filtered_df, columns)
+    print(f"df = {df}")
+    return df
+
+def search_table(df: pd.DataFrame, query: str) -> pd.DataFrame:
+    return df[(df[AutoEvalColumn.dummy.name].str.contains(query, case=False))]
+
+
+def select_columns(df: pd.DataFrame, columns: list) -> pd.DataFrame:
+    always_here_cols = [
+        AutoEvalColumn.model_type_symbol.name,
+        AutoEvalColumn.model.name,
+    ]
+    # We use COLS to maintain sorting
+    filtered_df = df[
+        always_here_cols + [c for c in COLS if c in df.columns and c in columns] + [AutoEvalColumn.dummy.name]
+    ]
+    return filtered_df
+
+def filter_queries(query: str, filtered_df: pd.DataFrame):
+    final_df = []
+    if query != "":
+        queries = [q.strip() for q in query.split(";")]
+        for _q in queries:
+            _q = _q.strip()
+            if _q != "":
+                temp_filtered_df = search_table(filtered_df, _q)
+                if len(temp_filtered_df) > 0:
+                    final_df.append(temp_filtered_df)
+        if len(final_df) > 0:
+            filtered_df = pd.concat(final_df)
+            filtered_df = filtered_df.drop_duplicates(
+                subset=[AutoEvalColumn.model.name, AutoEvalColumn.precision.name, AutoEvalColumn.revision.name]
+            )
+
+    return filtered_df
+
+
+def filter_models(df: pd.DataFrame, type_query: list, size_query: list, precision_query: list, show_deleted: bool) -> pd.DataFrame:
+
+    print(f"filter_models()'s df: {df}\n")
+    # Show all models
+    if show_deleted:
+        filtered_df = df
+    else:  # Show only still on the hub models
+        filtered_df = df[df[AutoEvalColumn.still_on_hub.name] is True]
+
+    type_emoji = [t[0] for t in type_query]
+    filtered_df = filtered_df.loc[df[AutoEvalColumn.model_type_symbol.name].isin(type_emoji)]
+    filtered_df = filtered_df.loc[df[AutoEvalColumn.precision.name].isin(precision_query + ["None"])]
+
+    numeric_interval = pd.IntervalIndex(sorted([NUMERIC_INTERVALS[s] for s in size_query]))
+    params_column = pd.to_numeric(df[AutoEvalColumn.params.name], errors="coerce")
+    mask = params_column.apply(lambda x: any(numeric_interval.contains(x)))
+    filtered_df = filtered_df.loc[mask]
+
+    return filtered_df
+
+
+## ------- ## ------- ## ------- ## ------- ## ------- ## ------- ## ------- ## ------- ## ------- ## ------- ## ------- ## ------- 
+
+ui_snapshot_download(repo_id=QUEUE_REPO, local_dir=EVAL_REQUESTS_PATH, repo_type="dataset", tqdm_class=None, etag_timeout=30)
+ui_snapshot_download(repo_id=RESULTS_REPO, local_dir=EVAL_RESULTS_PATH, repo_type="dataset", tqdm_class=None, etag_timeout=30)
+
+
+raw_data, original_df = get_leaderboard_df(EVAL_RESULTS_PATH, EVAL_REQUESTS_PATH, COLS, BENCHMARK_COLS) # k the problem is that the results are only saved in _bk dirs. 
+leaderboard_df = original_df.copy()
+
+
+################################################################################################################################
+demo = gr.Blocks(css=custom_css)
+with demo:
+    gr.HTML(TITLE)
+    gr.Markdown(INTRODUCTION_TEXT, elem_classes="markdown-text")
+
+    with gr.Tabs(elem_classes="tab-buttons") as tabs:
+
+        # toggle break 1: this tab just RENDERS existing result files on remote repo. 
+        with gr.TabItem("Benchmarks", elem_id="llm-benchmark-tab-table", id=0):
+        
+            with gr.Row():
+                with gr.Column():
+                    with gr.Row():
+                        search_bar = gr.Textbox(placeholder=" 🔍 Model search (separate multiple queries with `;`)", show_label=False, elem_id="search-bar",)
+                    with gr.Row():
+                        shown_columns = gr.CheckboxGroup(
+                            choices=[
+                                c.name
+                                for c in fields(AutoEvalColumn)
+                                if not c.hidden and not c.never_hidden and not c.dummy
+                            ],
+                            value=[
+                                c.name
+                                for c in fields(AutoEvalColumn)
+                                if c.displayed_by_default and not c.hidden and not c.never_hidden
+                            ],
+                            label="Select columns to show",
+                            elem_id="column-select",
+                            interactive=True,
+                        )
+
+                with gr.Column(min_width=320):
+                    filter_columns_type = gr.CheckboxGroup(
+                        label="Model types",
+                        choices=[t.to_str() for t in ModelType],
+                        value=[t.to_str() for t in ModelType],
+                        interactive=True,
+                        elem_id="filter-columns-type",
+                    )
+                    filter_columns_precision = gr.CheckboxGroup(
+                        label="Precision",
+                        choices=[i.value.name for i in Precision],
+                        value=[i.value.name for i in Precision],
+                        interactive=True,
+                        elem_id="filter-columns-precision",
+                    )
+                    filter_columns_size = gr.CheckboxGroup(
+                        label="Model sizes (in billions of parameters)",
+                        choices=list(NUMERIC_INTERVALS.keys()),
+                        value=list(NUMERIC_INTERVALS.keys()),
+                        interactive=True,
+                        elem_id="filter-columns-size",
+                    )
+
+            # leaderboard_table = gr.components.Dataframe(
+            #     value=leaderboard_df[
+            #         [c.name for c in fields(AutoEvalColumn) if c.never_hidden]
+            #         + shown_columns.value
+            #         + [AutoEvalColumn.dummy.name]
+            #     ] if leaderboard_df.empty is False else leaderboard_df,
+            #     headers=[c.name for c in fields(AutoEvalColumn) if c.never_hidden] + shown_columns.value,
+            #     datatype=TYPES,
+            #     elem_id="leaderboard-table",
+            #     interactive=False,
+            #     visible=True,
+            #     column_widths=["2%", "20%"]
+            # )
+            leaderboard_table = gr.components.Dataframe(
+                # value=leaderboard_df,
+                value=leaderboard_df[
+                    [c.name for c in fields(AutoEvalColumn) if c.never_hidden]
+                    + shown_columns.value
+                    + [AutoEvalColumn.dummy.name]
+                ] if leaderboard_df.empty is False else leaderboard_df,
+                headers=[c.name for c in fields(AutoEvalColumn) if c.never_hidden] + shown_columns.value,
+                datatype=TYPES,
+                elem_id="leaderboard-table",
+                interactive=False,
+                visible=True,
+                # column_widths=["2%", "20%"]
+            )
+            # Dummy leaderboard for handling the case when the user uses backspace key
+            hidden_leaderboard_table_for_search = gr.components.Dataframe(
+                value=original_df[COLS] if original_df.empty is False else original_df,
+                headers=COLS,
+                datatype=TYPES,
+                visible=False
+            )
+            for selector in [shown_columns, filter_columns_type, filter_columns_precision, filter_columns_size]:
+                selector.change(
+                    update_table,
+                    [
+                        hidden_leaderboard_table_for_search,
+                        shown_columns,
+                        filter_columns_type,
+                        filter_columns_precision,
+                        filter_columns_size,
+                        search_bar,
+                    ],
+                    leaderboard_table,
+                    queue=True,
+                )
+
+        # toggle break 2: Submission -> runs add_new_eval() (actual evaluation is done on backend when backend-cli.py is run.)
+        with gr.TabItem("🚀 Submit here! ", elem_id="llm-benchmark-tab-table", id=3):
+            # with gr.Column():
+            #     with gr.Row():
+            #         gr.Markdown(EVALUATION_QUEUE_TEXT, elem_classes="markdown-text")
+
+                # with gr.Column():
+                    # with gr.Accordion(
+                    #     f"✅ Finished Evaluations ({len(finished_eval_queue_df)})",
+                    #     open=False,
+                    # ):
+                    #     with gr.Row():
+                    #         finished_eval_table = gr.components.Dataframe(
+                    #             value=finished_eval_queue_df,
+                    #             headers=EVAL_COLS,
+                    #             datatype=EVAL_TYPES,
+                    #             row_count=5
+                    #         )
+                    # with gr.Accordion(
+                    #     f"🔄 Running Evaluation Queue ({len(running_eval_queue_df)})",
+                    #     open=False,
+                    # ):
+                    #     with gr.Row():
+                    #         running_eval_table = gr.components.Dataframe(
+                    #             value=running_eval_queue_df,
+                    #             headers=EVAL_COLS,
+                    #             datatype=EVAL_TYPES,
+                    #             row_count=5
+                    #         )
+
+                    # with gr.Accordion(
+                    #     f"⏳ Pending Evaluation Queue ({len(pending_eval_queue_df)})",
+                    #     open=False,
+                    # ):
+                    #     with gr.Row():
+                    #         pending_eval_table = gr.components.Dataframe(
+                    #             value=pending_eval_queue_df,
+                    #             headers=EVAL_COLS,
+                    #             datatype=EVAL_TYPES,
+                    #             row_count=5
+                    #         )
+            with gr.Row():
+                gr.Markdown("# ✉️✨ Submit your model here!", elem_classes="markdown-text")
+
+            with gr.Row():
+                with gr.Column():
+                    model_name_textbox = gr.Textbox(label="Model name")
+                    # You can use the revision parameter to point to the specific commit hash when downloading.
+                    revision_name_textbox = gr.Textbox(label="Revision commit", placeholder="main")
+                    private = gr.Checkbox(False, label="Private", visible=not IS_PUBLIC)
+                    model_type = gr.Dropdown(
+                        choices=[t.to_str(" : ") for t in ModelType if t != ModelType.Unknown],
+                        label="Model type",
+                        multiselect=False,
+                        value=None,
+                        interactive=True,
+                    )
+
+                with gr.Column():
+                    precision = gr.Dropdown(
+                        choices=[i.value.name for i in Precision if i != Precision.Unknown],
+                        label="Precision",
+                        multiselect=False,
+                        value="float32",
+                        interactive=True,
+                    )
+                    weight_type = gr.Dropdown(
+                        choices=[i.value.name for i in WeightType],
+                        label="Weights type",
+                        multiselect=False,
+                        value="Original",
+                        interactive=True,
+                    )
+
+
+                    requested_tasks = gr.CheckboxGroup(
+                        choices=[ (i.value.col_name, i.value) for i in Tasks], 
+
+                        label="Select tasks",
+                        elem_id="task-select",
+                        interactive=True,
+                    )
+
+
+                    base_model_name_textbox = gr.Textbox(label="Base model (for delta or adapter weights)")
+
+            submit_button = gr.Button("Submit Eval")
+            submission_result = gr.Markdown()
+
+            # we need to add task specification argument here as well. 
+            submit_button.click(
+                add_new_eval,
+                [
+                    model_name_textbox,
+
+                    requested_tasks, # is this a list of str or class Task? i think it's Task. 
+
+                    base_model_name_textbox,
+                    revision_name_textbox,
+                    precision,
+                    private,
+                    weight_type,
+                    model_type,
+                ],
+                submission_result,
+            )
+
+
+# demo.queue(default_concurrency_limit=40).launch()
+demo.launch()

app_empty.py

@@ -0,0 +1,7 @@
+import gradio as gr
+
+def greet(name):
+    return "Hello " + name + "!!"
+
+# iface = gr.Interface(fn=greet, inputs="text", outputs="text")
+# iface.launch()

backend-cli.py

@@ -0,0 +1,167 @@
+#!/usr/bin/env python
+
+import os
+import json
+
+import random
+from datetime import datetime
+
+from src.backend.run_eval_suite import run_evaluation
+from src.backend.manage_requests import check_completed_evals, get_eval_requests, set_eval_request
+from src.backend.sort_queue import sort_models_by_priority
+
+
+from src.backend.envs import EVAL_REQUESTS_PATH_BACKEND, EVAL_RESULTS_PATH_BACKEND, DEVICE, LIMIT, Tasks, Task, num_fewshots
+
+from src.backend.manage_requests import EvalRequest
+from src.leaderboard.read_evals import EvalResult
+
+from src.envs import QUEUE_REPO, RESULTS_REPO, API
+from src.utils import my_snapshot_download
+
+import time
+
+import logging
+import pprint
+
+
+def my_set_eval_request(api, eval_request, set_to_status, hf_repo, local_dir):
+    for i in range(10):
+        try:
+            set_eval_request(api=api, eval_request=eval_request, set_to_status=set_to_status, hf_repo=hf_repo, local_dir=local_dir)
+            return
+        except Exception:
+            time.sleep(60)
+    return
+
+
+logging.getLogger("openai").setLevel(logging.WARNING)
+
+logging.basicConfig(level=logging.ERROR)
+pp = pprint.PrettyPrinter(width=80)
+
+PENDING_STATUS = "PENDING"
+RUNNING_STATUS = "RUNNING"
+FINISHED_STATUS = "FINISHED"
+FAILED_STATUS = "FAILED"
+
+TASKS_HARNESS = [task.value for task in Tasks]
+
+# starts by downloading results and requests. makes sense since we want to be able to use different backend servers!
+my_snapshot_download(repo_id=RESULTS_REPO, revision="main", local_dir=EVAL_RESULTS_PATH_BACKEND, repo_type="dataset", max_workers=60)
+my_snapshot_download(repo_id=QUEUE_REPO, revision="main", local_dir=EVAL_REQUESTS_PATH_BACKEND, repo_type="dataset", max_workers=60)
+
+
+def sanity_checks():
+    print(f'Device: {DEVICE}')
+
+    # pull the eval dataset from the hub and parse any eval requests
+    # check completed evals and set them to finished
+    my_snapshot_download(repo_id=QUEUE_REPO, revision="main", local_dir=EVAL_REQUESTS_PATH_BACKEND, repo_type="dataset", max_workers=60)
+    check_completed_evals(api=API, checked_status=RUNNING_STATUS, completed_status=FINISHED_STATUS,
+                          failed_status=FAILED_STATUS, hf_repo=QUEUE_REPO, local_dir=EVAL_REQUESTS_PATH_BACKEND,
+                          hf_repo_results=RESULTS_REPO, local_dir_results=EVAL_RESULTS_PATH_BACKEND)
+    return
+
+
+def request_to_result_name(request: EvalRequest) -> str:
+    
+    org_and_model = request.model.split("/", 1)
+    if len(org_and_model) == 1:
+        model = org_and_model[0]
+        res = f"{model}_{request.precision}"
+    else:
+        org = org_and_model[0]
+        model = org_and_model[1]
+        res = f"{org}_{model}_{request.precision}"
+    return res
+
+# doesn't make distinctions for tasks since the original code runs eval on ALL tasks. 
+def process_evaluation(task_name: str, eval_request: EvalRequest) -> dict:
+    # batch_size = 1
+    batch_size = "auto"
+
+    # might not have to get the benchmark. 
+    print(f"task_name parameter in process_evaluation() = {task_name}") #, task_names=[task.benchmark] = {[task.benchmark]}")
+
+    num_fewshot = num_fewshots[task_name]
+
+    results = run_evaluation(eval_request=eval_request, task_names=task_name, num_fewshot=num_fewshot,
+                             batch_size=batch_size, device=DEVICE, use_cache=None, limit=LIMIT)
+
+    print('RESULTS', results)
+
+    dumped = json.dumps(results, indent=2, default=lambda o: '<not serializable>')
+    print(dumped)
+
+    output_path = os.path.join(EVAL_RESULTS_PATH_BACKEND, *eval_request.model.split("/"), f"results_{task_name}_{datetime.now()}.json")
+    os.makedirs(os.path.dirname(output_path), exist_ok=True)
+    with open(output_path, "w") as f:
+        f.write(dumped)
+
+    my_snapshot_download(repo_id=RESULTS_REPO, revision="main", local_dir=EVAL_RESULTS_PATH_BACKEND, repo_type="dataset", max_workers=60)
+    API.upload_file(path_or_fileobj=output_path, path_in_repo=f"{eval_request.model}/results_{task_name}_{datetime.now()}.json",
+                    repo_id=RESULTS_REPO, repo_type="dataset")
+    return results
+
+
+# the rendering is done with files in local repo. 
+def process_pending_requests() -> bool:
+    sanity_checks()
+
+    current_pending_status = [PENDING_STATUS]
+
+    # Get all eval request that are PENDING, if you want to run other evals, change this parameter
+    # GETTING REQUESTS FROM THE HUB NOT LOCAL DIR. 
+    eval_requests = get_eval_requests(job_status=current_pending_status, hf_repo=QUEUE_REPO, local_dir=EVAL_REQUESTS_PATH_BACKEND)
+    # Sort the evals by priority (first submitted first run)
+    eval_requests = sort_models_by_priority(api=API, models=eval_requests)
+
+    random.shuffle(eval_requests)
+
+    # this says zero 
+    print(f"Found {len(eval_requests)} {','.join(current_pending_status)} eval requests")
+
+    if len(eval_requests) == 0:
+        return False
+
+    eval_request = eval_requests[0] 
+    pp.pprint(eval_request)
+
+    my_snapshot_download(repo_id=QUEUE_REPO, revision="main", local_dir=EVAL_REQUESTS_PATH_BACKEND, repo_type="dataset", max_workers=60)
+    my_set_eval_request(api=API, eval_request=eval_request, set_to_status=RUNNING_STATUS, hf_repo=QUEUE_REPO, local_dir=EVAL_REQUESTS_PATH_BACKEND)
+
+    # task_lst = TASKS_HARNESS.copy()
+    task_lst = eval_request.get_user_requested_task_names()
+    random.shuffle(task_lst)
+    print(f"task_lst in process_pending_requests(): {task_lst}")
+
+    for task_name in task_lst: 
+        
+        results = process_evaluation(task_name, eval_request)
+
+    my_snapshot_download(repo_id=QUEUE_REPO, revision="main", local_dir=EVAL_REQUESTS_PATH_BACKEND, repo_type="dataset", max_workers=60)
+    my_set_eval_request(api=API, eval_request=eval_request, set_to_status=FINISHED_STATUS, hf_repo=QUEUE_REPO, local_dir=EVAL_REQUESTS_PATH_BACKEND)
+
+    return True
+
+
+if __name__ == "__main__":
+    # wait = True
+
+    # import socket
+    # if socket.gethostname() in {'hamburg'} or os.path.isdir("/home/pminervi"):
+    #     wait = False
+
+    # if wait:
+    #     time.sleep(60 * random.randint(2, 5))
+    #     pass
+
+    # res = False
+    res = process_pending_requests()
+
+    # if res is False:
+    #     res = process_finished_requests(100)
+
+    # if res is False:
+    #     res = process_finished_requests(0)

eval-queue-bk/.gitattributes

@@ -0,0 +1,55 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.lz4 filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tar filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text
+# Audio files - uncompressed
+*.pcm filter=lfs diff=lfs merge=lfs -text
+*.sam filter=lfs diff=lfs merge=lfs -text
+*.raw filter=lfs diff=lfs merge=lfs -text
+# Audio files - compressed
+*.aac filter=lfs diff=lfs merge=lfs -text
+*.flac filter=lfs diff=lfs merge=lfs -text
+*.mp3 filter=lfs diff=lfs merge=lfs -text
+*.ogg filter=lfs diff=lfs merge=lfs -text
+*.wav filter=lfs diff=lfs merge=lfs -text
+# Image files - uncompressed
+*.bmp filter=lfs diff=lfs merge=lfs -text
+*.gif filter=lfs diff=lfs merge=lfs -text
+*.png filter=lfs diff=lfs merge=lfs -text
+*.tiff filter=lfs diff=lfs merge=lfs -text
+# Image files - compressed
+*.jpg filter=lfs diff=lfs merge=lfs -text
+*.jpeg filter=lfs diff=lfs merge=lfs -text
+*.webp filter=lfs diff=lfs merge=lfs -text

eval-queue-bk/EleutherAI/pythia-160m_eval_request_False_float32_Original.json

@@ -0,0 +1 @@
+{"model": "EleutherAI/pythia-160m", "requested_tasks": [{"benchmark": "hellaswag", "metric": "acc_norm", "col_name": "HellaSwag"}], "base_model": "", "revision": "main", "private": false, "precision": "float32", "weight_type": "Original", "status": "FINISHED", "submitted_time": "2024-02-01T00:29:51Z", "model_type": "\ud83d\udfe2 : pretrained", "likes": 17, "params": 0.213, "license": "apache-2.0"}

eval-queue-bk/EleutherAI/pythia-410m_eval_request_False_float32_Original.json

@@ -0,0 +1 @@
+{"model": "EleutherAI/pythia-410m", "requested_tasks": [{"benchmark": "hellaswag", "metric": "acc_norm", "col_name": "HellaSwag"}, {"benchmark": "pubmedqa", "metric": "acc", "col_name": "PubMedQA"}], "base_model": "", "revision": "main", "private": false, "precision": "float32", "weight_type": "Original", "status": "FINISHED", "submitted_time": "2024-02-01T02:54:14Z", "model_type": "\ud83d\udfe2 : pretrained", "likes": 16, "params": 0.506, "license": "apache-2.0"}

eval-queue-bk/EleutherAI/pythia-70m_eval_request_False_float32_Original.json

@@ -0,0 +1 @@
+{"model": "EleutherAI/pythia-70m", "requested_tasks": [{"benchmark": "hellaswag", "metric": "acc_norm", "col_name": "HellaSwag"}], "base_model": "", "revision": "main", "private": false, "precision": "float32", "weight_type": "Original", "status": "FINISHED", "submitted_time": "2024-02-01T02:42:51Z", "model_type": "\ud83d\udfe2 : pretrained", "likes": 32, "params": 0.096, "license": "apache-2.0"}

eval-queue-bk/README.md

@@ -0,0 +1,3 @@
+---
+license: mit
+---

eval-queue/.gitattributes

@@ -0,0 +1,55 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.lz4 filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tar filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text
+# Audio files - uncompressed
+*.pcm filter=lfs diff=lfs merge=lfs -text
+*.sam filter=lfs diff=lfs merge=lfs -text
+*.raw filter=lfs diff=lfs merge=lfs -text
+# Audio files - compressed
+*.aac filter=lfs diff=lfs merge=lfs -text
+*.flac filter=lfs diff=lfs merge=lfs -text
+*.mp3 filter=lfs diff=lfs merge=lfs -text
+*.ogg filter=lfs diff=lfs merge=lfs -text
+*.wav filter=lfs diff=lfs merge=lfs -text
+# Image files - uncompressed
+*.bmp filter=lfs diff=lfs merge=lfs -text
+*.gif filter=lfs diff=lfs merge=lfs -text
+*.png filter=lfs diff=lfs merge=lfs -text
+*.tiff filter=lfs diff=lfs merge=lfs -text
+# Image files - compressed
+*.jpg filter=lfs diff=lfs merge=lfs -text
+*.jpeg filter=lfs diff=lfs merge=lfs -text
+*.webp filter=lfs diff=lfs merge=lfs -text

eval-queue/EleutherAI/pythia-160m_eval_request_False_float32_Original.json

@@ -0,0 +1 @@
+{"model": "EleutherAI/pythia-160m", "requested_tasks": [{"benchmark": "hellaswag", "metric": "acc_norm", "col_name": "HellaSwag"}], "base_model": "", "revision": "main", "private": false, "precision": "float32", "weight_type": "Original", "status": "FINISHED", "submitted_time": "2024-02-01T00:29:51Z", "model_type": "\ud83d\udfe2 : pretrained", "likes": 17, "params": 0.213, "license": "apache-2.0"}

eval-queue/EleutherAI/pythia-410m_eval_request_False_float32_Original.json

@@ -0,0 +1 @@
+{"model": "EleutherAI/pythia-410m", "requested_tasks": [{"benchmark": "hellaswag", "metric": "acc_norm", "col_name": "HellaSwag"}, {"benchmark": "pubmedqa", "metric": "acc", "col_name": "PubMedQA"}], "base_model": "", "revision": "main", "private": false, "precision": "float32", "weight_type": "Original", "status": "FINISHED", "submitted_time": "2024-02-01T02:54:14Z", "model_type": "\ud83d\udfe2 : pretrained", "likes": 16, "params": 0.506, "license": "apache-2.0"}

eval-queue/EleutherAI/pythia-70m_eval_request_False_float32_Original.json

@@ -0,0 +1 @@
+{"model": "EleutherAI/pythia-70m", "requested_tasks": [{"benchmark": "hellaswag", "metric": "acc_norm", "col_name": "HellaSwag"}], "base_model": "", "revision": "main", "private": false, "precision": "float32", "weight_type": "Original", "status": "FINISHED", "submitted_time": "2024-02-01T02:42:51Z", "model_type": "\ud83d\udfe2 : pretrained", "likes": 32, "params": 0.096, "license": "apache-2.0"}

eval-queue/README.md

@@ -0,0 +1,3 @@
+---
+license: mit
+---

eval-results-bk/.gitattributes

@@ -0,0 +1,55 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.lz4 filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tar filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text
+# Audio files - uncompressed
+*.pcm filter=lfs diff=lfs merge=lfs -text
+*.sam filter=lfs diff=lfs merge=lfs -text
+*.raw filter=lfs diff=lfs merge=lfs -text
+# Audio files - compressed
+*.aac filter=lfs diff=lfs merge=lfs -text
+*.flac filter=lfs diff=lfs merge=lfs -text
+*.mp3 filter=lfs diff=lfs merge=lfs -text
+*.ogg filter=lfs diff=lfs merge=lfs -text
+*.wav filter=lfs diff=lfs merge=lfs -text
+# Image files - uncompressed
+*.bmp filter=lfs diff=lfs merge=lfs -text
+*.gif filter=lfs diff=lfs merge=lfs -text
+*.png filter=lfs diff=lfs merge=lfs -text
+*.tiff filter=lfs diff=lfs merge=lfs -text
+# Image files - compressed
+*.jpg filter=lfs diff=lfs merge=lfs -text
+*.jpeg filter=lfs diff=lfs merge=lfs -text
+*.webp filter=lfs diff=lfs merge=lfs -text

eval-results-bk/README.md

@@ -0,0 +1,3 @@
+---
+license: mit
+---

eval-results/.gitattributes

@@ -0,0 +1,55 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.lz4 filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tar filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text
+# Audio files - uncompressed
+*.pcm filter=lfs diff=lfs merge=lfs -text
+*.sam filter=lfs diff=lfs merge=lfs -text
+*.raw filter=lfs diff=lfs merge=lfs -text
+# Audio files - compressed
+*.aac filter=lfs diff=lfs merge=lfs -text
+*.flac filter=lfs diff=lfs merge=lfs -text
+*.mp3 filter=lfs diff=lfs merge=lfs -text
+*.ogg filter=lfs diff=lfs merge=lfs -text
+*.wav filter=lfs diff=lfs merge=lfs -text
+# Image files - uncompressed
+*.bmp filter=lfs diff=lfs merge=lfs -text
+*.gif filter=lfs diff=lfs merge=lfs -text
+*.png filter=lfs diff=lfs merge=lfs -text
+*.tiff filter=lfs diff=lfs merge=lfs -text
+# Image files - compressed
+*.jpg filter=lfs diff=lfs merge=lfs -text
+*.jpeg filter=lfs diff=lfs merge=lfs -text
+*.webp filter=lfs diff=lfs merge=lfs -text

eval-results/README.md

@@ -0,0 +1,3 @@
+---
+license: mit
+---

manage_repos.ipynb

@@ -0,0 +1,176 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "https://huggingface.co/datasets/chaeeunlee/test_requests\n",
+    "\n",
+    "https://huggingface.co/datasets/chaeeunlee/test_results"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 18,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "'\\n( path_in_repo: str\\nrepo_id: str\\ntoken: typing.Optional[str] = None\\nrepo_type: typing.Optional[str] = Nonerevision: typing.Optional[str] = Nonecommit_message: typing.Optional[str] = Nonecommit_description: typing.Optional[str] = Nonecreate_pr: typing.Optional[bool] = Noneparent_commit: typing.Optional[str] = None )\\n'"
+      ]
+     },
+     "execution_count": 18,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "from src.envs import H4_TOKEN, API, QUEUE_REPO, RESULTS_REPO\n",
+    "\n",
+    "from huggingface_hub import HfApi\n",
+    "\n",
+    "'''\n",
+    "( path_in_repo: str\n",
+    "repo_id: str\n",
+    "token: typing.Optional[str] = None\n",
+    "repo_type: typing.Optional[str] = Nonerevision: typing.Optional[str] = Nonecommit_message: typing.Optional[str] = Nonecommit_description: typing.Optional[str] = Nonecreate_pr: typing.Optional[bool] = Noneparent_commit: typing.Optional[str] = None )\n",
+    "'''\n",
+    "\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 38,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "res = API.delete_folder(path_in_repo='EleutherAI/', repo_id=RESULTS_REPO, repo_type='dataset')"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 37,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "res = API.delete_folder(path_in_repo='EleutherAI/', repo_id=QUEUE_REPO, repo_type='dataset')"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# import os\n",
+    "\n",
+    "# for root, _, files in os.walk(results_path):\n",
+    "#     # We should only have json files in model results\n",
+    "#     if len(files) == 0 or any([not f.endswith(\".json\") for f in files]):\n",
+    "#         continue\n",
+    "\n",
+    "#     # Sort the files by date\n",
+    "#     try:\n",
+    "#         files.sort(key=lambda x: x.removesuffix(\".json\").removeprefix(\"results_\")[:-7])\n",
+    "#     except dateutil.parser._parser.ParserError:\n",
+    "#         files = [files[-1]]\n",
+    "\n",
+    "\n",
+    "#     print(f\"files = {files}\")\n",
+    "\n",
+    "#     for file in files:\n",
+    "#         model_result_filepaths.append(os.path.join(root, file))"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "DatasetInfo(id='chaeeunlee/test_requests', author='chaeeunlee', sha='c7f4d0c0b1207cc773dcd0b1df49cd6a883e02be', created_at=datetime.datetime(2024, 1, 31, 11, 19, 22, tzinfo=datetime.timezone.utc), last_modified=datetime.datetime(2024, 1, 31, 19, 55, 30, tzinfo=datetime.timezone.utc), private=False, gated=False, disabled=False, downloads=0, likes=0, paperswithcode_id=None, tags=['license:mit', 'region:us'], card_data={'annotations_creators': None, 'language_creators': None, 'language': None, 'license': 'mit', 'multilinguality': None, 'size_categories': None, 'source_datasets': None, 'task_categories': None, 'task_ids': None, 'paperswithcode_id': None, 'pretty_name': None, 'config_names': None, 'train_eval_index': None}, siblings=[RepoSibling(rfilename='.gitattributes', size=None, blob_id=None, lfs=None), RepoSibling(rfilename='EleutherAI/pythia-160m_eval_request_False_float32_Original.json', size=None, blob_id=None, lfs=None), RepoSibling(rfilename='README.md', size=None, blob_id=None, lfs=None)])\n"
+     ]
+    }
+   ],
+   "source": [
+    "info = API.dataset_info(repo_id=QUEUE_REPO)\n",
+    "print(info)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 21,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from huggingface_hub import HfApi\n",
+    "\n",
+    "def print_repo_directory_structure(api, repo_id, is_dataset=True):\n",
+    "    \"\"\"\n",
+    "    Print the directory structure of a Hugging Face repository.\n",
+    "\n",
+    "    Parameters:\n",
+    "    - repo_id (str): Repository ID in the format \"username/reponame\".\n",
+    "    \"\"\"\n",
+    "    # api = HfApi()\n",
+    "    if is_dataset:\n",
+    "        repo_files = api.list_repo_files(repo_id=repo_id, repo_type='dataset')\n",
+    "    else:\n",
+    "        repo_files = api.list_repo_files(repo_id=repo_id)\n",
+    "\n",
+    "\n",
+    "    print(f\"Directory structure of {repo_id}:\")\n",
+    "    print()\n",
+    "    for file_path in repo_files:\n",
+    "        print(file_path)\n",
+    "\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 35,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Directory structure of chaeeunlee/test_requests:\n",
+      "\n",
+      ".gitattributes\n",
+      "README.md\n"
+     ]
+    }
+   ],
+   "source": [
+    "repo_id = QUEUE_REPO  # Replace with the target repository ID\n",
+    "print_repo_directory_structure(API, repo_id)"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "lb",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.10.13"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}

requirements.txt

@@ -0,0 +1,31 @@
+torch
+colorama
+APScheduler
+black
+click
+datasets
+gradio
+gradio_client
+huggingface-hub
+matplotlib
+numpy
+pandas
+plotly
+python-dateutil
+requests
+semantic-version
+tqdm
+transformers>=4.36.0,<4.37.0
+tokenizers>=0.15.0
+lm_eval[ifeval] @ git+https://github.com/EleutherAI/lm-evaluation-harness.git
+accelerate
+sentencepiece
+langdetect
+sacrebleu
+cchardet
+rouge_score
+bert-score
+evaluate
+spacy
+selfcheckgpt
+immutabledict