ruff format everything

app.py

@@ -24,39 +24,33 @@ from src.envs import (
 )
 from src.leaderboard.build_leaderboard import build_leadearboard_df
 
-os.environ['GRADIO_ANALYTICS_ENABLED']='false'
+os.environ["GRADIO_ANALYTICS_ENABLED"] = "false"
 
 # Configure logging
-logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')
+logging.basicConfig(level=logging.INFO, format="%(asctime)s - %(levelname)s - %(message)s")
 
 # Start ephemeral Spaces on PRs (see config in README.md)
 enable_space_ci()
 
+
 def restart_space():
     API.restart_space(repo_id=REPO_ID, token=H4_TOKEN)
 
 
 def build_demo():
-    demo = gr.Blocks(
-        title = "Chatbot Arena Leaderboard",
-        css=custom_css
-    )
+    demo = gr.Blocks(title="Chatbot Arena Leaderboard", css=custom_css)
     leaderboard_df = build_leadearboard_df()
     with demo:
         gr.HTML(TITLE)
         gr.Markdown(INTRODUCTION_TEXT, elem_classes="markdown-text")
 
-        with gr.Tabs(elem_classes="tab-buttons") as tabs:
+        with gr.Tabs(elem_classes="tab-buttons"):
             with gr.TabItem("🏅 LLM Benchmark", elem_id="llm-benchmark-tab-table", id=0):
-                leaderboard = Leaderboard(
+                Leaderboard(
                     value=leaderboard_df,
                     datatype=[c.type for c in fields(AutoEvalColumn)],
                     select_columns=SelectColumns(
-                        default_selection=[
-                            c.name
-                            for c in fields(AutoEvalColumn)
-                            if c.displayed_by_default
-                        ],
+                        default_selection=[c.name for c in fields(AutoEvalColumn) if c.displayed_by_default],
                         cant_deselect=[c.name for c in fields(AutoEvalColumn) if c.never_hidden or c.dummy],
                         label="Select Columns to Display:",
                     ),
@@ -67,50 +61,59 @@ def build_demo():
                     ],
                 )
 
-            #with gr.TabItem("📝 About", elem_id="llm-benchmark-tab-table", id=1):
+            # with gr.TabItem("📝 About", elem_id="llm-benchmark-tab-table", id=1):
             #    gr.Markdown(LLM_BENCHMARKS_TEXT, elem_classes="markdown-text")
-            #with gr.TabItem("❗FAQ", elem_id="llm-benchmark-tab-table", id=2):
+            # with gr.TabItem("❗FAQ", elem_id="llm-benchmark-tab-table", id=2):
             #    gr.Markdown(FAQ_TEXT, elem_classes="markdown-text")
 
             with gr.TabItem("🚀 Submit ", elem_id="llm-benchmark-tab-table", id=3):
-
                 with gr.Row():
                     gr.Markdown("# ✨ Submit your model here!", elem_classes="markdown-text")
 
                 with gr.Column():
-                        model_name_textbox = gr.Textbox(label="Model name")
-                        def upload_file(file):
-                            file_path = file.name.split('/')[-1] if '/' in file.name else file.name
-                            logging.info("New submition: file saved to %s", file_path)
-                            API.upload_file(path_or_fileobj=file.name,path_in_repo='./external/'+file_path,repo_id='Vikhrmodels/openbench-eval',repo_type='dataset')
-                            os.environ[RESET_JUDGEMENT_ENV] = '1'
-                            return file.name
-                        
-                        if model_name_textbox:
-                            file_output = gr.File()
-                            upload_button = gr.UploadButton("Click to Upload & Submit Answers", file_types=['*'], file_count="single")
-                            upload_button.upload(upload_file, upload_button, file_output)
-                            
+                    model_name_textbox = gr.Textbox(label="Model name")
+
+                    def upload_file(file):
+                        file_path = file.name.split("/")[-1] if "/" in file.name else file.name
+                        logging.info("New submition: file saved to %s", file_path)
+                        API.upload_file(
+                            path_or_fileobj=file.name,
+                            path_in_repo="./external/" + file_path,
+                            repo_id="Vikhrmodels/openbench-eval",
+                            repo_type="dataset",
+                        )
+                        os.environ[RESET_JUDGEMENT_ENV] = "1"
+                        return file.name
+
+                    if model_name_textbox:
+                        file_output = gr.File()
+                        upload_button = gr.UploadButton(
+                            "Click to Upload & Submit Answers", file_types=["*"], file_count="single"
+                        )
+                        upload_button.upload(upload_file, upload_button, file_output)
+
         return demo
-             
+
+
 # print(os.system('cd src/gen && ../../.venv/bin/python gen_judgment.py'))
 # print(os.system('cd src/gen/ && python show_result.py --output'))
-           
+
+
 def update_board():
     need_reset = os.environ.get(RESET_JUDGEMENT_ENV)
-    if need_reset != '1':
+    if need_reset != "1":
         return
-    os.environ[RESET_JUDGEMENT_ENV] = '0'
-    subprocess.run(['python', 'src/gen/gen_judgement.py'], check = False)
-    subprocess.Popen('python3.src/gen/show_result.py --output')
+    os.environ[RESET_JUDGEMENT_ENV] = "0"
+    subprocess.run(["python", "src/gen/gen_judgement.py"], check=False)
+    subprocess.Popen("python3.src/gen/show_result.py --output")
 
 
 if __name__ == "__main__":
-    os.environ[RESET_JUDGEMENT_ENV] = '1'
-    
+    os.environ[RESET_JUDGEMENT_ENV] = "1"
+
     scheduler = BackgroundScheduler()
-    scheduler.add_job(update_board, "interval", minutes=10) 
+    scheduler.add_job(update_board, "interval", minutes=10)
     scheduler.start()
-    
+
     demo_app = build_demo()
     demo_app.launch(debug=True)

src/display/css_html_js.py

@@ -88,4 +88,4 @@ get_window_url_params = """
         url_params = Object.fromEntries(params);
         return url_params;
     }
-    """
+    """

src/display/utils.py

@@ -7,7 +7,8 @@ import pandas as pd
 
 
 # Configure logging
-logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')
+logging.basicConfig(level=logging.INFO, format="%(asctime)s - %(levelname)s - %(message)s")
+
 
 def parse_datetime(datetime_str):
     formats = [
@@ -25,6 +26,7 @@ def parse_datetime(datetime_str):
     logging.error(f"No valid date format found for: {datetime_str}")
     return datetime(1970, 1, 1)
 
+
 def load_json_data(file_path):
     """Safely load JSON data from a file."""
     try:
@@ -98,7 +100,6 @@ auto_eval_column_dict.append(["score", ColumnContent, ColumnContent("score", "nu
 AutoEvalColumn = make_dataclass("AutoEvalColumn", auto_eval_column_dict, frozen=True)
 
 
-
 @dataclass(frozen=True)
 class EvalQueueColumn:  # Queue column
     model = ColumnContent("model", "markdown", True)

src/gen/gen_answer.py

@@ -33,7 +33,14 @@ from utils import (
 
 
 def get_answer(
-    question: dict, model: str, endpoint_info: dict, num_choices: int, max_tokens: int, temperature: float, answer_file: str, api_dict: dict
+    question: dict,
+    model: str,
+    endpoint_info: dict,
+    num_choices: int,
+    max_tokens: int,
+    temperature: float,
+    answer_file: str,
+    api_dict: dict,
 ):
     if question["category"] in temperature_config:
         temperature = temperature_config[question["category"]]
@@ -54,49 +61,56 @@ def get_answer(
         for j in range(len(question["turns"])):
             conv.append({"role": "user", "content": question["turns"][j]["content"]})
             if api_type == "anthropic":
-                output = chat_completion_anthropic(model=endpoint_info["model_name"],
-                                                   messages=conv,
-                                                   temperature=temperature,
-                                                   max_tokens=max_tokens)
+                output = chat_completion_anthropic(
+                    model=endpoint_info["model_name"], messages=conv, temperature=temperature, max_tokens=max_tokens
+                )
             elif api_type == "mistral":
-                output = chat_completion_mistral(model=endpoint_info["model_name"],
-                                                 messages=conv,
-                                                 temperature=temperature,
-                                                 max_tokens=max_tokens)
+                output = chat_completion_mistral(
+                    model=endpoint_info["model_name"], messages=conv, temperature=temperature, max_tokens=max_tokens
+                )
             elif api_type == "yandex":
-                output = chat_completion_yandex(model=endpoint_info["model_name"],
-                                                 messages=conv,
-                                                 temperature=temperature,
-                                                 max_tokens=max_tokens,
-                                                 api_dict=api_dict)
+                output = chat_completion_yandex(
+                    model=endpoint_info["model_name"],
+                    messages=conv,
+                    temperature=temperature,
+                    max_tokens=max_tokens,
+                    api_dict=api_dict,
+                )
             elif api_type == "gigachat":
-                output = chat_completion_gigachat(model=endpoint_info["model_name"],
-                                                 messages=conv,
-                                                 temperature=temperature,
-                                                 max_tokens=max_tokens,
-                                                 api_dict=api_dict)
+                output = chat_completion_gigachat(
+                    model=endpoint_info["model_name"],
+                    messages=conv,
+                    temperature=temperature,
+                    max_tokens=max_tokens,
+                    api_dict=api_dict,
+                )
             elif api_type == "gemini":
-                output = chat_completion_gemini(model=endpoint_info["model_name"],
-                                                messages=question["turns"][j]["content"],
-                                                temperature=temperature,
-                                                max_tokens=max_tokens)
+                output = chat_completion_gemini(
+                    model=endpoint_info["model_name"],
+                    messages=question["turns"][j]["content"],
+                    temperature=temperature,
+                    max_tokens=max_tokens,
+                )
             elif api_type == "azure":
-                output = chat_completion_openai_azure(model=endpoint_info["model_name"],
-                                                      messages=conv,
-                                                      temperature=temperature,
-                                                      max_tokens=max_tokens,
-                                                      api_dict=api_dict)
+                output = chat_completion_openai_azure(
+                    model=endpoint_info["model_name"],
+                    messages=conv,
+                    temperature=temperature,
+                    max_tokens=max_tokens,
+                    api_dict=api_dict,
+                )
             elif api_type == "cohere":
-                output = chat_completion_cohere(model=endpoint_info["model_name"],
-                                                messages=conv,
-                                                temperature=temperature,
-                                                max_tokens=max_tokens)
+                output = chat_completion_cohere(
+                    model=endpoint_info["model_name"], messages=conv, temperature=temperature, max_tokens=max_tokens
+                )
             else:
-                output = chat_completion_openai(model=endpoint_info["model_name"],
-                                                messages=conv,
-                                                temperature=temperature,
-                                                max_tokens=max_tokens,
-                                                api_dict=api_dict)
+                output = chat_completion_openai(
+                    model=endpoint_info["model_name"],
+                    messages=conv,
+                    temperature=temperature,
+                    max_tokens=max_tokens,
+                    api_dict=api_dict,
+                )
             conv.append({"role": "assistant", "content": output})
 
             turns.append({"content": output, "token_len": len(encoding.encode(output))})
@@ -118,12 +132,8 @@ def get_answer(
 
 if __name__ == "__main__":
     parser = argparse.ArgumentParser()
-    parser.add_argument(
-        "--setting-file", type=str, default="config/gen_answer_config.yaml"
-    )
-    parser.add_argument(
-        "--endpoint-file", type=str, default="config/api_config.yaml"
-    )
+    parser.add_argument("--setting-file", type=str, default="config/gen_answer_config.yaml")
+    parser.add_argument("--endpoint-file", type=str, default="config/api_config.yaml")
     args = parser.parse_args()
 
     settings = make_config(args.setting_file)
@@ -187,9 +197,7 @@ if __name__ == "__main__":
                 futures.append(future)
             if count > 0:
                 print(f"{count} number of existing answers")
-            for future in tqdm.tqdm(
-                concurrent.futures.as_completed(futures), total=len(futures)
-            ):
+            for future in tqdm.tqdm(concurrent.futures.as_completed(futures), total=len(futures)):
                 future.result()
 
         reorg_answer_file(answer_file)

src/gen/gen_judgment.py

@@ -55,12 +55,7 @@ def judgment(**args):
 
     num_games = 2 if configs["pairwise"] else 1
 
-    output = {
-        "question_id":question["question_id"],
-        "model":answer["model_id"],
-        "judge": model,
-        "games":[]
-        }
+    output = {"question_id": question["question_id"], "model": answer["model_id"], "judge": model, "games": []}
 
     for game in range(num_games):
         conv = [{"role": "system", "content": configs["system_prompt"]}]
@@ -73,7 +68,7 @@ def judgment(**args):
             base = 1
 
             if baseline:
-                if game % 2 == 1: # swap position
+                if game % 2 == 1:  # swap position
                     temp = baseline
                     baseline = answer
                     answer = temp
@@ -103,7 +98,7 @@ def judgment(**args):
                 args["endpoint_dict"],
             )
 
-            judgment += ("\n" + new_judgment)
+            judgment += "\n" + new_judgment
 
             score, try_again = get_score(judgment, args["regex_pattern"])
 
@@ -112,18 +107,21 @@ def judgment(**args):
             if not try_again:
                 break
 
-            conv.append({"role": "user", "content": "continue your judgment and finish by outputting a final verdict label"})
+            conv.append(
+                {"role": "user", "content": "continue your judgment and finish by outputting a final verdict label"}
+            )
 
-        result = {
-            "user_prompt": conv[1]["content"],
-            "judgment": judgment,
-            "score":score
-        }
+        result = {"user_prompt": conv[1]["content"], "judgment": judgment, "score": score}
         output["games"].append(result)
 
     with open(output_file, "a") as f:
         f.write(json.dumps(output, ensure_ascii=False) + "\n")
-    huggingface_hub.HfApi().upload_file(output_file, path_in_repo=f'model_judgment/{configs['judge_model']}/{output_file.split('/')[-1]}', repo_id='Vikhrmodels/openbench-eval', repo_type='dataset')
+    huggingface_hub.HfApi().upload_file(
+        output_file,
+        path_in_repo=f'model_judgment/{configs['judge_model']}/{output_file.split('/')[-1]}',
+        repo_id="Vikhrmodels/openbench-eval",
+        repo_type="dataset",
+    )
 
 
 if __name__ == "__main__":
@@ -136,8 +134,10 @@ if __name__ == "__main__":
     configs = make_config(args.setting_file)
     endpoint_list = make_config(args.endpoint_file)
 
-    print(f'judge model: {configs["judge_model"]}, baseline: {configs["baseline"]}, baseline model: {configs["baseline_model"]}, reference: {configs["reference"]}, '
-          + f'reference models: {configs["ref_model"]}, temperature: {configs["temperature"]}, max tokens: {configs["max_tokens"]}, pairwise: {configs["pairwise"]}')
+    print(
+        f'judge model: {configs["judge_model"]}, baseline: {configs["baseline"]}, baseline model: {configs["baseline_model"]}, reference: {configs["reference"]}, '
+        + f'reference models: {configs["ref_model"]}, temperature: {configs["temperature"]}, max tokens: {configs["max_tokens"]}, pairwise: {configs["pairwise"]}'
+    )
 
     if configs["regex_pattern"]:
         pattern = re.compile(configs["regex_pattern"])
@@ -150,12 +150,15 @@ if __name__ == "__main__":
     questions = load_questions(question_file)
     model_answers_external = load_model_answers(external_dir)
     model_answers_internal = load_model_answers(internal_dir)
-    
+
     # internal has priority
     model_answers = {**model_answers_external, **model_answers_internal}
 
     # if user choose a set of models, only judge those models
-    models = [model.split('/')[-1].split('.')[0] for model in glob.glob('./data/arena-hard-v0.1/model_answer/external/*.jsonl')]
+    models = [
+        model.split("/")[-1].split(".")[0]
+        for model in glob.glob("./data/arena-hard-v0.1/model_answer/external/*.jsonl")
+    ]
 
     ref_answers = None
     if configs["reference"]:
@@ -214,7 +217,5 @@ if __name__ == "__main__":
             if count > 0:
                 print(f"{count} number of existing judgments")
 
-        for future in tqdm(
-            concurrent.futures.as_completed(futures), total=len(futures)
-        ):
+        for future in tqdm(concurrent.futures.as_completed(futures), total=len(futures)):
             future.result()

src/gen/show_result.py

@@ -2,7 +2,6 @@ import pandas as pd
 import numpy as np
 import plotly.express as px
 
-import tiktoken
 import datetime
 import argparse
 import os
@@ -15,6 +14,7 @@ from sklearn.linear_model import LogisticRegression
 from collections import defaultdict
 from utils import load_model_answers
 
+
 def compute_mle_elo(df, SCALE=400, BASE=10, INIT_RATING=1000):
     models = pd.concat([df["model_a"], df["model_b"]]).unique()
     models = pd.Series(np.arange(len(models)), index=models)
@@ -35,18 +35,18 @@ def compute_mle_elo(df, SCALE=400, BASE=10, INIT_RATING=1000):
     # one tie => one A win + one B win
     # find tie + tie (both bad) index
     tie_idx = (df["winner"] == "tie") | (df["winner"] == "tie (bothbad)")
-    tie_idx[len(tie_idx)//2:] = False
+    tie_idx[len(tie_idx) // 2 :] = False
     Y[tie_idx] = 1.0
 
     lr = LogisticRegression(fit_intercept=False, penalty=None, tol=1e-8)
-    lr.fit(X,Y)
+    lr.fit(X, Y)
 
     elo_scores = SCALE * lr.coef_[0] + INIT_RATING
 
     # set anchor as gpt-3.5-turbo-0125 = 1000
     if "gpt-3.5-turbo-0125" in models.index:
         elo_scores += 1000 - elo_scores[models["gpt-3.5-turbo-0125"]]
-    return pd.Series(elo_scores, index = models.index).sort_values(ascending=False)
+    return pd.Series(elo_scores, index=models.index).sort_values(ascending=False)
 
 
 def get_bootstrap_result(battles, func_compute_elo, num_round):
@@ -58,9 +58,14 @@ def get_bootstrap_result(battles, func_compute_elo, num_round):
 
 
 def preety_print_two_ratings(ratings_1, ratings_2, column_names):
-    df = pd.DataFrame([
-        [n, ratings_1[n], ratings_2[n]] for n in ratings_1.keys()
-    ], columns=["Model", column_names[0], column_names[1]]).sort_values(column_names[0], ascending=False).reset_index(drop=True)
+    df = (
+        pd.DataFrame(
+            [[n, ratings_1[n], ratings_2[n]] for n in ratings_1.keys()],
+            columns=["Model", column_names[0], column_names[1]],
+        )
+        .sort_values(column_names[0], ascending=False)
+        .reset_index(drop=True)
+    )
     df[column_names[0]] = (df[column_names[0]] + 0.5).astype(int)
     df[column_names[1]] = (df[column_names[1]] + 0.5).astype(int)
     df.index = df.index + 1
@@ -68,18 +73,24 @@ def preety_print_two_ratings(ratings_1, ratings_2, column_names):
 
 
 def visualize_bootstrap_scores(df, title):
-    bars = pd.DataFrame(dict(
-        lower = df.quantile(.025),
-        rating = df.quantile(.5),
-        upper = df.quantile(.975))).reset_index(names="model").sort_values("rating", ascending=False)
-    bars['error_y'] = bars['upper'] - bars["rating"]
-    bars['error_y_minus'] = bars['rating'] - bars["lower"]
-    bars['rating_rounded'] = np.round(bars['rating'], 2)
-    fig = px.scatter(bars, x="model", y="rating", error_y="error_y",
-                     error_y_minus="error_y_minus", text="rating_rounded",
-                     title=title)
-    fig.update_layout(xaxis_title="Model", yaxis_title="Rating",
-                      height=600)
+    bars = (
+        pd.DataFrame(dict(lower=df.quantile(0.025), rating=df.quantile(0.5), upper=df.quantile(0.975)))
+        .reset_index(names="model")
+        .sort_values("rating", ascending=False)
+    )
+    bars["error_y"] = bars["upper"] - bars["rating"]
+    bars["error_y_minus"] = bars["rating"] - bars["lower"]
+    bars["rating_rounded"] = np.round(bars["rating"], 2)
+    fig = px.scatter(
+        bars,
+        x="model",
+        y="rating",
+        error_y="error_y",
+        error_y_minus="error_y_minus",
+        text="rating_rounded",
+        title=title,
+    )
+    fig.update_layout(xaxis_title="Model", yaxis_title="Rating", height=600)
     return fig
 
 
@@ -92,10 +103,7 @@ def predict_win_rate(elo_ratings, SCALE=400, BASE=10, INIT_RATING=1000):
             wins[a][b] = ea
             wins[b][a] = 1 - ea
 
-    data = {
-        a: [wins[a][b] if a != b else np.NAN for b in names]
-        for a in names
-    }
+    data = {a: [wins[a][b] if a != b else np.NAN for b in names] for a in names}
 
     df = pd.DataFrame(data, index=names)
     df.index.name = "model_a"
@@ -121,9 +129,7 @@ def get_battles_from_judgment(judge_name, first_game_only=False, WEIGHT=3):
 
         for _, row in df.iterrows():
             # game 1
-            output = {"question_id": row["question_id"],
-                    "model_a": "gpt-3.5-turbo-0125",
-                    "model_b": row["model"]}
+            output = {"question_id": row["question_id"], "model_a": "gpt-3.5-turbo-0125", "model_b": row["model"]}
 
             game = row["games"][0]
 
@@ -148,9 +154,7 @@ def get_battles_from_judgment(judge_name, first_game_only=False, WEIGHT=3):
 
             if not first_game_only:
                 # game 2
-                output = {"question_id": row["question_id"],
-                        "model_a": "gpt-3.5-turbo-0125",
-                        "model_b": row["model"]}
+                output = {"question_id": row["question_id"], "model_a": "gpt-3.5-turbo-0125", "model_b": row["model"]}
 
                 game = row["games"][1]
 
@@ -190,7 +194,9 @@ if __name__ == "__main__":
     parser.add_argument("--first-game-only", action="store_true")
     args = parser.parse_args()
     print(args)
-    assert not args.load_bootstrap or (args.load_battles and args.load_bootstrap), "If loading prexisting bootstrapping data, you must also load preexisting battles."
+    assert not args.load_bootstrap or (
+        args.load_battles and args.load_bootstrap
+    ), "If loading prexisting bootstrapping data, you must also load preexisting battles."
 
     answer_dir = os.path.join("data", args.bench_name, "model_answer/external")
     model_answers = load_model_answers(answer_dir)
@@ -203,7 +209,6 @@ if __name__ == "__main__":
 
     bootstrap_online_elo = compute_mle_elo(battles)
 
-
     if args.load_bootstrap:
         bootstrap_elo_lu = pd.read_json("data/bootstrapping_results.jsonl", lines=True)
     else:
@@ -213,7 +218,7 @@ if __name__ == "__main__":
 
     stats = pd.DataFrame()
     stats["results"] = None
-    stats["results"] = stats['results'].astype('object')
+    stats["results"] = stats["results"].astype("object")
 
     for i, model in enumerate(bootstrap_online_elo.index):
         assert model in bootstrap_elo_lu.columns
@@ -241,18 +246,24 @@ if __name__ == "__main__":
         decimal = 1
     else:
         decimal = 0
-        stats = stats.astype({"score" : int, "lower" : int, "upper" : int})
+        stats = stats.astype({"score": int, "lower": int, "upper": int})
 
     stats.sort_values(by="score", ascending=False, inplace=True)
     for _, row in stats.iterrows():
-        interval = str((round(row['lower'] - row['score'], decimal), round(row['upper'] - row['score'], decimal)))
-        print(f"{row['model'] : <30} | score: {round(row['score'], decimal) : ^5} | 95% CI: {interval : ^12} | average #tokens: {int(row['avg_tokens'])}")
+        interval = str((round(row["lower"] - row["score"], decimal), round(row["upper"] - row["score"], decimal)))
+        print(
+            f"{row['model'] : <30} | score: {round(row['score'], decimal) : ^5} | 95% CI: {interval : ^12} | average #tokens: {int(row['avg_tokens'])}"
+        )
 
     if args.output:
         cur_date = datetime.datetime.now()
         date_str = cur_date.strftime("%Y%m%d")
         stats.to_json(f"arena_hard_leaderboard_{date_str}.json", orient="records", indent=4)
         import huggingface_hub
-        huggingface_hub.HfApi().upload_file(path_or_fileobj=f"arena_hard_leaderboard_{date_str}.json",path_in_repo='evals/upd.json',
-                                          repo_id='Vikhrmodels/openbench-eval',
-                                          repo_type='dataset')
+
+        huggingface_hub.HfApi().upload_file(
+            path_or_fileobj=f"arena_hard_leaderboard_{date_str}.json",
+            path_in_repo="evals/upd.json",
+            repo_id="Vikhrmodels/openbench-eval",
+            repo_type="dataset",
+        )

src/gen/utils.py

@@ -77,9 +77,7 @@ def get_endpoint(endpoint_list):
         return None
     assert endpoint_list is not None
     # randomly pick one
-    api_dict = random.choices(
-        endpoint_list
-    )[0]
+    api_dict = random.choices(endpoint_list)[0]
     return api_dict
 
 
@@ -91,9 +89,11 @@ def make_config(config_file: str) -> dict:
 
     return config_kwargs
 
+
 def chat_completion_gigachat(model, messages, temperature, max_tokens, api_dict=None):
     from gigachat import GigaChat
     from gigachat.models import Chat, Messages
+
     assert api_dict is not None, "no api settings provided!"
     auth_token = api_dict.get("auth_token", os.environ.get(api_dict["auth_token"], ""))
     client = GigaChat(credentials=auth_token, model=model, verify_ssl_certs=False)
@@ -115,15 +115,13 @@ def chat_completion_gigachat(model, messages, temperature, max_tokens, api_dict=
 
     return output
 
+
 def chat_completion_yandex(model, messages, temperature, max_tokens, api_dict=None):
     from yandex_gpt import YandexGPT, YandexGPTConfigManagerForIAMToken
+
     assert api_dict is not None, "no api settings provided!"
     iam_token = api_dict.get("iam_token", os.environ.get(api_dict["iam_token_ENV"], ""))
-    config = YandexGPTConfigManagerForIAMToken(
-        model_type=model,
-        catalog_id=api_dict["catalog_id"],
-        iam_token=iam_token
-    )
+    config = YandexGPTConfigManagerForIAMToken(model_type=model, catalog_id=api_dict["catalog_id"], iam_token=iam_token)
     client = YandexGPT(config_manager=config)
 
     messages = [{"role": m["role"], "text": m["content"]} for m in messages]
@@ -147,6 +145,7 @@ def chat_completion_yandex(model, messages, temperature, max_tokens, api_dict=No
 
 def chat_completion_openai(model, messages, temperature, max_tokens, api_dict=None):
     import openai
+
     api_key = api_dict.get("api_key", os.environ.get(api_dict["api_key_ENV"], ""))
     if api_dict:
         client = openai.OpenAI(
@@ -165,8 +164,8 @@ def chat_completion_openai(model, messages, temperature, max_tokens, api_dict=No
                 messages=messages,
                 temperature=temperature,
                 max_tokens=max_tokens,
-                stop=["</s>", "<eos>", "<|eot_id|>"]
-                )
+                stop=["</s>", "<eos>", "<|eot_id|>"],
+            )
             output = completion.choices[0].message.content
             break
         except openai.RateLimitError as e:
@@ -175,7 +174,7 @@ def chat_completion_openai(model, messages, temperature, max_tokens, api_dict=No
         except openai.BadRequestError as e:
             print(messages)
             print(type(e), e)
-        except KeyError:
+        except KeyError as e:
             print(type(e), e)
             break
 
@@ -189,11 +188,7 @@ def chat_completion_openai_azure(model, messages, temperature, max_tokens, api_d
     api_base = api_dict["api_base"]
     api_key = api_dict.get("api_key", os.environ.get(api_dict["api_key_ENV"], ""))
     client = AzureOpenAI(
-        azure_endpoint = api_base,
-        api_key= api_key,
-        api_version=api_dict["api_version"],
-        timeout=240,
-        max_retries=2
+        azure_endpoint=api_base, api_key=api_key, api_version=api_dict["api_version"], timeout=240, max_retries=2
     )
 
     output = API_ERROR_OUTPUT
@@ -215,7 +210,7 @@ def chat_completion_openai_azure(model, messages, temperature, max_tokens, api_d
         except openai.BadRequestError as e:
             print(type(e), e)
             break
-        except KeyError:
+        except KeyError as e:
             print(type(e), e)
             break
 
@@ -246,7 +241,7 @@ def chat_completion_anthropic(model, messages, temperature, max_tokens, api_dict
                 stop_sequences=[anthropic.HUMAN_PROMPT],
                 max_tokens=max_tokens,
                 temperature=temperature,
-                system=sys_msg
+                system=sys_msg,
             )
             output = response.content[0].text
             break
@@ -286,25 +281,14 @@ def chat_completion_mistral(model, messages, temperature, max_tokens):
 
 def chat_completion_gemini(model, messages, temperature, max_tokens):
     import google.generativeai as genai
+
     genai.configure(api_key=os.environ["GEMINI_API_KEY"])
 
     safety_settings = [
-        {
-            "category": "HARM_CATEGORY_HARASSMENT",
-            "threshold": "BLOCK_NONE"
-        },
-        {
-            "category": "HARM_CATEGORY_HATE_SPEECH",
-            "threshold": "BLOCK_NONE"
-        },
-        {
-            "category": "HARM_CATEGORY_SEXUALLY_EXPLICIT",
-            "threshold": "BLOCK_NONE"
-        },
-        {
-            "category": "HARM_CATEGORY_DANGEROUS_CONTENT",
-            "threshold": "BLOCK_NONE"
-        },
+        {"category": "HARM_CATEGORY_HARASSMENT", "threshold": "BLOCK_NONE"},
+        {"category": "HARM_CATEGORY_HATE_SPEECH", "threshold": "BLOCK_NONE"},
+        {"category": "HARM_CATEGORY_SEXUALLY_EXPLICIT", "threshold": "BLOCK_NONE"},
+        {"category": "HARM_CATEGORY_DANGEROUS_CONTENT", "threshold": "BLOCK_NONE"},
     ]
 
     # Set up the model
@@ -319,9 +303,8 @@ def chat_completion_gemini(model, messages, temperature, max_tokens):
     for _ in range(API_MAX_RETRY):
         try:
             gemini = genai.GenerativeModel(
-                model_name=model,
-                generation_config=generation_config,
-                safety_settings=safety_settings)
+                model_name=model, generation_config=generation_config, safety_settings=safety_settings
+            )
 
             convo = gemini.start_chat(history=[])
 
@@ -344,9 +327,7 @@ def chat_completion_cohere(model, messages, temperature, max_tokens):
     co = cohere.Client(os.environ["COHERE_API_KEY"])
     assert len(messages) > 0
 
-    template_map = {"system":"SYSTEM",
-                    "assistant":"CHATBOT",
-                    "user":"USER"}
+    template_map = {"system": "SYSTEM", "assistant": "CHATBOT", "user": "USER"}
 
     assert messages[-1]["role"] == "user"
     prompt = messages[-1]["content"]
@@ -354,7 +335,7 @@ def chat_completion_cohere(model, messages, temperature, max_tokens):
     if len(messages) > 1:
         history = []
         for message in messages[:-1]:
-            history.append({"role":template_map[message["role"]], "message":message["content"]})
+            history.append({"role": template_map[message["role"]], "message": message["content"]})
     else:
         history = None
 
@@ -384,9 +365,9 @@ def reorg_answer_file(answer_file):
     """Sort by question id and de-duplication"""
     answers = {}
     with open(answer_file, "r") as fin:
-        for l in fin:
-            qid = json.loads(l)["question_id"]
-            answers[qid] = l
+        for line in fin:
+            qid = json.loads(line)["question_id"]
+            answers[qid] = line
 
     qids = sorted(list(answers.keys()))
     with open(answer_file, "w") as fout:

src/leaderboard/build_leaderboard.py

@@ -1,4 +1,3 @@
-
 import json
 import logging
 import os
@@ -11,7 +10,8 @@ from huggingface_hub import snapshot_download
 from src.envs import EVAL_RESULTS_PATH
 
 # Configure logging
-logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')
+logging.basicConfig(level=logging.INFO, format="%(asctime)s - %(levelname)s - %(message)s")
+
 
 def time_diff_wrapper(func):
     def wrapper(*args, **kwargs):
@@ -21,15 +21,17 @@ def time_diff_wrapper(func):
         diff = end_time - start_time
         logging.info(f"Time taken for {func.__name__}: {diff} seconds")
         return result
+
     return wrapper
 
+
 @time_diff_wrapper
 def download_dataset(repo_id, local_dir, repo_type="dataset", max_attempts=3, backoff_factor=1.5):
     """Download dataset with exponential backoff retries."""
     attempt = 0
     while attempt < max_attempts:
         try:
-            logging.info(f"Downloading {repo_id} to {local_dir}")
+            logging.info("Downloading %s to %s", repo_id, local_dir)
             snapshot_download(
                 repo_id=repo_id,
                 local_dir=local_dir,
@@ -42,27 +44,41 @@ def download_dataset(repo_id, local_dir, repo_type="dataset", max_attempts=3, ba
             logging.info("Download successful")
             return
         except Exception as e:
-            wait_time = backoff_factor ** attempt
+            wait_time = backoff_factor**attempt
             logging.error(f"Error downloading {repo_id}: {e}, retrying in {wait_time}s")
             time.sleep(wait_time)
             attempt += 1
     logging.error(f"Failed to download {repo_id} after {max_attempts} attempts")
 
+
 def build_leadearboard_df():
     """Initializes the application space, loading only necessary data."""
-    # Check ENV LEADERBOARD_DOWNLOAD if wee need to download the leaderboard
-    if os.getenv("LEADERBOARD_DOWNLOAD", "True") == "True":
-        # These downloads only occur on full initialization
-        # try:
-            # download_dataset(QUEUE_REPO, EVAL_REQUESTS_PATH)
-            # download_dataset(DYNAMIC_INFO_REPO, DYNAMIC_INFO_PATH)
-            download_dataset("Vikhrmodels/openbench-eval", EVAL_RESULTS_PATH)
-            # print(subprocess.Popen('ls src'))
-            subprocess.run(['rsync', '-avzP', '--ignore-existing', f'{EVAL_RESULTS_PATH[2:]}/external/*', 'src/gen/data/arena-hard-v0.1/model_answer/'], check=False)
-            subprocess.run(['rsync', '-avzP', '--ignore-existing', f'{EVAL_RESULTS_PATH[2:]}/model_judgment/*', 'src/gen/data/arena-hard-v0.1/model_judgement/'], check=False)
-        # except Exception:
-        #     restart_space()
 
-    # Always retrieve the leaderboard DataFrame
-    leaderboard_df = pd.DataFrame.from_records(json.load(open('eval-results/evals/upd.json','r')))
+    # download_dataset(QUEUE_REPO, EVAL_REQUESTS_PATH)
+    # download_dataset(DYNAMIC_INFO_REPO, DYNAMIC_INFO_PATH)
+    download_dataset("Vikhrmodels/openbench-eval", EVAL_RESULTS_PATH)
+    # print(subprocess.Popen('ls src'))
+    subprocess.run(
+        [
+            "rsync",
+            "-avzP",
+            "--ignore-existing",
+            f"{EVAL_RESULTS_PATH}/external/*",
+            "src/gen/data/arena-hard-v0.1/model_answer/",
+        ],
+        check=False,
+    )
+    subprocess.run(
+        [
+            "rsync",
+            "-avzP",
+            "--ignore-existing",
+            f"{EVAL_RESULTS_PATH}/model_judgment/*",
+            "src/gen/data/arena-hard-v0.1/model_judgement/",
+        ],
+        check=False,
+    )
+
+    # Retrieve the leaderboard DataFrame
+    leaderboard_df = pd.DataFrame.from_records(json.load(open("eval-results/evals/upd.json", "r")))
     return leaderboard_df.copy()

src/leaderboard/filter_models.py

@@ -137,9 +137,9 @@ def flag_models(leaderboard_data: list[dict]):
         if model_data[AutoEvalColumn.not_flagged.name]:
             flag_key = model_data[AutoEvalColumn.fullname.name]
         else:
-             # Merges and moes are flagged 
+            # Merges and moes are flagged
             flag_key = "merged"
-        
+
         # Reverse the logic: Check for non-flagged models instead
         if flag_key in FLAGGED_MODELS:
             issue_num = FLAGGED_MODELS[flag_key].split("/")[-1]
@@ -147,9 +147,9 @@ def flag_models(leaderboard_data: list[dict]):
                 FLAGGED_MODELS[flag_key],
                 f"See discussion #{issue_num}",
             )
-            model_data[AutoEvalColumn.model.name] = (
-                f"{model_data[AutoEvalColumn.model.name]} has been flagged! {issue_link}"
-            )
+            model_data[
+                AutoEvalColumn.model.name
+            ] = f"{model_data[AutoEvalColumn.model.name]} has been flagged! {issue_link}"
             model_data[AutoEvalColumn.not_flagged.name] = False
         else:
             model_data[AutoEvalColumn.not_flagged.name] = True
@@ -171,4 +171,3 @@ def remove_forbidden_models(leaderboard_data: list[dict]):
 def filter_models_flags(leaderboard_data: list[dict]):
     leaderboard_data = remove_forbidden_models(leaderboard_data)
     flag_models(leaderboard_data)
-

src/leaderboard/read_evals.py

@@ -16,36 +16,36 @@ from src.display.formatting import make_clickable_model
 from src.display.utils import AutoEvalColumn, ModelType, Precision, Tasks, WeightType, parse_datetime
 
 # Configure logging
-logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')
+logging.basicConfig(level=logging.INFO, format="%(asctime)s - %(levelname)s - %(message)s")
+
 
 @dataclass
 class EvalResult:
     # Also see src.display.utils.AutoEvalColumn for what will be displayed.
-    eval_name: str # org_model_precision (uid)
-    full_model: str # org/model (path on hub)
+    eval_name: str  # org_model_precision (uid)
+    full_model: str  # org/model (path on hub)
     org: Optional[str]
     model: str
-    revision: str # commit hash, "" if main
+    revision: str  # commit hash, "" if main
     results: Dict[str, float]
     precision: Precision = Precision.Unknown
-    model_type: ModelType = ModelType.Unknown # Pretrained, fine tuned, ...
+    model_type: ModelType = ModelType.Unknown  # Pretrained, fine tuned, ...
     weight_type: WeightType = WeightType.Original
-    architecture: str = "Unknown" # From config file
+    architecture: str = "Unknown"  # From config file
     license: str = "?"
     likes: int = 0
     num_params: int = 0
-    date: str = "" # submission date of request file
+    date: str = ""  # submission date of request file
     still_on_hub: bool = True
     is_merge: bool = False
     not_flagged: bool = False
     status: str = "FINISHED"
     # List of tags, initialized to a new empty list for each instance to avoid the pitfalls of mutable default arguments.
     tags: List[str] = field(default_factory=list)
- 
-    
+
     @classmethod
-    def init_from_json_file(cls, json_filepath: str) -> 'EvalResult':
-        with open(json_filepath, 'r') as fp:
+    def init_from_json_file(cls, json_filepath: str) -> "EvalResult":
+        with open(json_filepath, "r") as fp:
             data = json.load(fp)
 
         config = data.get("config_general", {})
@@ -72,7 +72,7 @@ class EvalResult:
             model=model,
             results=results,
             precision=precision,
-            revision=config.get("model_sha", "")
+            revision=config.get("model_sha", ""),
         )
 
     @staticmethod
@@ -118,9 +118,8 @@ class EvalResult:
 
             mean_acc = np.mean(accs) * 100.0
             results[task.benchmark] = mean_acc
-        
-        return results
 
+        return results
 
     def update_with_request_file(self, requests_path):
         """Finds the relevant request file for the current model and updates info with it."""
@@ -130,17 +129,17 @@ class EvalResult:
                 logging.warning(f"No request file for {self.org}/{self.model}")
                 self.status = "FAILED"
                 return
-            
+
             with open(request_file, "r") as f:
                 request = json.load(f)
-            
+
             self.model_type = ModelType.from_str(request.get("model_type", "Unknown"))
             self.weight_type = WeightType[request.get("weight_type", "Original")]
             self.num_params = int(request.get("params", 0))  # Ensuring type safety
             self.date = request.get("submitted_time", "")
             self.architecture = request.get("architectures", "Unknown")
             self.status = request.get("status", "FAILED")
-            
+
         except FileNotFoundError:
             self.status = "FAILED"
             logging.error(f"Request file: {request_file} not found for {self.org}/{self.model}")
@@ -154,7 +153,6 @@ class EvalResult:
             self.status = "FAILED"
             logging.error(f"Unexpected error {e} for {self.org}/{self.model}")
 
-
     def update_with_dynamic_file_dict(self, file_dict):
         """Update object attributes based on the provided dictionary, with error handling for missing keys and type validation."""
         # Default values set for optional or potentially missing keys.
@@ -162,11 +160,10 @@ class EvalResult:
         self.likes = int(file_dict.get("likes", 0))  # Ensure likes is treated as an integer
         self.still_on_hub = file_dict.get("still_on_hub", False)  # Default to False if key is missing
         self.tags = file_dict.get("tags", [])
-        
+
         # Calculate `flagged` only if 'tags' is not empty and avoid calculating each time
         self.not_flagged = not (any("flagged" in tag for tag in self.tags))
 
-
     def to_dict(self):
         """Converts the Eval Result to a dict compatible with our dataframe display"""
         average = sum([v for v in self.results.values() if v is not None]) / len(Tasks)
@@ -185,8 +182,10 @@ class EvalResult:
             AutoEvalColumn.likes.name: self.likes,
             AutoEvalColumn.params.name: self.num_params,
             AutoEvalColumn.still_on_hub.name: self.still_on_hub,
-            AutoEvalColumn.merged.name: not( "merge" in self.tags if self.tags else False),
-            AutoEvalColumn.moe.name: not ( ("moe" in self.tags if self.tags else False) or "moe" in self.full_model.lower()) ,
+            AutoEvalColumn.merged.name: not ("merge" in self.tags if self.tags else False),
+            AutoEvalColumn.moe.name: not (
+                ("moe" in self.tags if self.tags else False) or "moe" in self.full_model.lower()
+            ),
             AutoEvalColumn.not_flagged.name: self.not_flagged,
         }
 
@@ -194,16 +193,16 @@ class EvalResult:
             data_dict[task.value.col_name] = self.results[task.value.benchmark]
 
         return data_dict
-    
+
 
 def get_request_file_for_model(requests_path, model_name, precision):
     """Selects the correct request file for a given model. Only keeps runs tagged as FINISHED"""
     requests_path = Path(requests_path)
     pattern = f"{model_name}_eval_request_*.json"
-    
+
     # Using pathlib to find files matching the pattern
     request_files = list(requests_path.glob(pattern))
-    
+
     # Sort the files by name in descending order to mimic 'reverse=True'
     request_files.sort(reverse=True)
 
@@ -214,7 +213,7 @@ def get_request_file_for_model(requests_path, model_name, precision):
             req_content = json.load(f)
             if req_content["status"] == "FINISHED" and req_content["precision"] == precision.split(".")[-1]:
                 request_file = str(request_file)
-    
+
     # Return empty string if no file found that matches criteria
     return request_file
 
@@ -223,9 +222,9 @@ def get_raw_eval_results(results_path: str, requests_path: str, dynamic_path: st
     """From the path of the results folder root, extract all needed info for results"""
     with open(dynamic_path) as f:
         dynamic_data = json.load(f)
-    
+
     results_path = Path(results_path)
-    model_files = list(results_path.rglob('results_*.json'))
+    model_files = list(results_path.rglob("results_*.json"))
     model_files.sort(key=lambda file: parse_datetime(file.stem.removeprefix("results_")))
 
     eval_results = {}
@@ -260,4 +259,3 @@ def get_raw_eval_results(results_path: str, requests_path: str, dynamic_path: st
             continue
 
     return results
-

src/populate.py

@@ -1,5 +1,3 @@
-import json
-import os
 import pathlib
 import pandas as pd
 from src.display.formatting import has_no_nan_values, make_clickable_model
@@ -21,7 +19,7 @@ def get_evaluation_queue_df(save_path, cols):
     save_path = pathlib.Path(save_path)
     all_evals = []
 
-    for path in save_path.rglob('*.json'):
+    for path in save_path.rglob("*.json"):
         data = load_json_data(path)
         if data:
             all_evals.append(_process_model_data(data))

src/scripts/create_request_file.py

@@ -47,7 +47,7 @@ def main():
     eval_entry = {
         "model": model_name,
         "base_model": base_model,
-        "revision": model_info.sha, # force to use the exact model commit 
+        "revision": model_info.sha,  # force to use the exact model commit
         "private": False,
         "precision": precision,
         "weight_type": weight_type,

src/scripts/update_all_request_files.py

@@ -91,6 +91,6 @@ def update_models(file_path, models_on_the_hub):
 
 def update_dynamic_files():
     # from gen import gen_answer,gen_judgment\
-    subprocess.Popen('python3 ../gen/gen_judgement.py')
+    subprocess.Popen("python3 ../gen/gen_judgement.py")
 
-    subprocess.Popen('python3 ../gen/show_result.py --output')
+    subprocess.Popen("python3 ../gen/show_result.py --output")

src/submission/check_validity.py

@@ -49,7 +49,7 @@ def is_model_on_hub(
         )  # , force_download=True)
         if test_tokenizer:
             try:
-                tk = AutoTokenizer.from_pretrained(
+                AutoTokenizer.from_pretrained(
                     model_name, revision=revision, trust_remote_code=trust_remote_code, token=token
                 )
             except ValueError as e:

src/submission/submit.py

@@ -1,21 +1,4 @@
-import json
-import os
-from datetime import datetime, timezone
-
-from huggingface_hub import snapshot_download
-
-from src.display.formatting import styled_error, styled_message, styled_warning
-from src.envs import (
-    API,
-    DYNAMIC_INFO_FILE_PATH,
-    DYNAMIC_INFO_PATH,
-    DYNAMIC_INFO_REPO,
-    EVAL_REQUESTS_PATH,
-    H4_TOKEN,
-    QUEUE_REPO,
-    RATE_LIMIT_PERIOD,
-    RATE_LIMIT_QUOTA,
-)
+from src.display.formatting import styled_message
 # from src.leaderboard.filter_models import DO_NOT_SUBMIT_MODELS
 # from src.submission.check_validity import (
 #     already_submitted_models,
@@ -38,7 +21,6 @@ def add_new_eval(
     # if not REQUESTED_MODELS:
     #     REQUESTED_MODELS, USERS_TO_SUBMISSION_DATES = already_submitted_models(EVAL_REQUESTS_PATH)
 
-
     # user_name = ""
     # model_path = model
     # if "/" in model:
@@ -186,6 +168,4 @@ def add_new_eval(
     # # Remove the local file
     # os.remove(out_path)
 
-    return styled_message(
-        "Your request has been submitted to the evaluation queue!\nPlease wait for up to an hour."
-    )
+    return styled_message("Your request has been submitted to the evaluation queue!\nPlease wait for up to an hour.")

src/tools/plots.py

@@ -3,7 +3,7 @@ import pandas as pd
 import plotly.express as px
 from plotly.graph_objs import Figure
 
-from src.display.utils import  AutoEvalColumn, Task, Tasks
+from src.display.utils import AutoEvalColumn, Task, Tasks
 from src.display.utils import human_baseline_row as HUMAN_BASELINE
 from src.leaderboard.filter_models import FLAGGED_MODELS
 from src.leaderboard.read_evals import EvalResult