add private repo

.gitignore

@@ -10,6 +10,8 @@ gpt_4_evals/
 human_evals/
 eval-queue/
 eval-results/
+eval-queue-private/
+eval-results-private/
 auto_evals/
 
 src/assets/model_counts.html

app.py

@@ -2,6 +2,7 @@ import json
 import os
 from datetime import datetime, timezone
 import re
+from distutils.util import strtobool
 
 import gradio as gr
 import pandas as pd
@@ -38,10 +39,10 @@ H4_TOKEN = os.environ.get("H4_TOKEN", None)
 QUEUE_REPO = "open-ko-llm-leaderboard/requests"
 RESULTS_REPO = "open-ko-llm-leaderboard/results"
 
-PRIVATE_QUEUE_REPO = "open-ko-llm-leaderboard/requests"
-PRIVATE_RESULTS_REPO = "open-ko-llm-leaderboard/results"
+PRIVATE_QUEUE_REPO = "open-ko-llm-leaderboard/private-requests"
+PRIVATE_RESULTS_REPO = "open-ko-llm-leaderboard/private-results"
 
-IS_PUBLIC = bool(os.environ.get("IS_PUBLIC", True))
+IS_PUBLIC = bool(strtobool(os.environ.get("IS_PUBLIC", "True")))
 
 EVAL_REQUESTS_PATH = "eval-queue"
 EVAL_RESULTS_PATH = "eval-results"

src/assets/hardcoded_evals.py

@@ -1,31 +1,5 @@
 from src.display_models.utils import AutoEvalColumn, model_hyperlink
 
-gpt4_values = {
-    AutoEvalColumn.model.name: model_hyperlink("https://arxiv.org/abs/2303.08774", "gpt4"),
-    AutoEvalColumn.revision.name: "tech report",
-    AutoEvalColumn.precision.name: None,
-    AutoEvalColumn.average.name: 84.3,
-    AutoEvalColumn.arc.name: 96.3,
-    AutoEvalColumn.hellaswag.name: 95.3,
-    AutoEvalColumn.mmlu.name: 86.4,
-    AutoEvalColumn.truthfulqa.name: 59.0,
-    AutoEvalColumn.dummy.name: "GPT-4",
-    AutoEvalColumn.model_type.name: "",
-}
-
-gpt35_values = {
-    AutoEvalColumn.model.name: model_hyperlink("https://arxiv.org/abs/2303.08774", "gpt3.5"),
-    AutoEvalColumn.revision.name: "tech report",
-    AutoEvalColumn.precision.name: None,
-    AutoEvalColumn.average.name: 71.9,
-    AutoEvalColumn.arc.name: 85.2,
-    AutoEvalColumn.hellaswag.name: 85.5,
-    AutoEvalColumn.mmlu.name: 70.0,
-    AutoEvalColumn.truthfulqa.name: 47.0,
-    AutoEvalColumn.dummy.name: "GPT-3.5",
-    AutoEvalColumn.model_type.name: "",
-}
-
 baseline = {
     AutoEvalColumn.model.name: "<p>Baseline</p>",
     AutoEvalColumn.revision.name: "N/A",

src/display_models/read_results.py

@@ -2,6 +2,7 @@ import json
 import os
 from dataclasses import dataclass
 from typing import Dict, List, Tuple
+from distutils.util import strtobool
 
 import dateutil
 import numpy as np
@@ -19,7 +20,7 @@ BENCH_TO_NAME = {
     # TODO: Uncomment when we have results for these
     # "ethicalverification": AutoEvalColumn.ethicalverification.name,
 }
-
+IS_PUBLIC = bool(strtobool(os.environ.get("IS_PUBLIC", "True")))
 
 @dataclass
 class EvalResult:
@@ -114,7 +115,7 @@ def parse_eval_result(json_filepath: str) -> Tuple[str, list[dict]]:
 def get_eval_results() -> List[EvalResult]:
     json_filepaths = []
 
-    for root, dir, files in os.walk("eval-results"):
+    for root, dir, files in os.walk("eval-results" + ("-private" if not IS_PUBLIC else "")):
         # We should only have json files in model results
         if len(files) == 0 or any([not f.endswith(".json") for f in files]):
             continue

src/load_from_hub.py

@@ -6,13 +6,11 @@ from huggingface_hub import Repository
 from transformers import AutoConfig
 from collections import defaultdict
 
-from src.assets.hardcoded_evals import baseline, gpt4_values, gpt35_values
+from src.assets.hardcoded_evals import baseline
 from src.display_models.get_model_metadata import apply_metadata
 from src.display_models.read_results import get_eval_results_dicts, make_clickable_model
 from src.display_models.utils import AutoEvalColumn, EvalQueueColumn, has_no_nan_values
 
-IS_PUBLIC = bool(os.environ.get("IS_PUBLIC", True))
-
 
 def get_all_requested_models(requested_models_dir: str) -> set[str]:
     depth = 1
@@ -75,10 +73,6 @@ def get_leaderboard_df(
 
     all_data = get_eval_results_dicts()
 
-    if not IS_PUBLIC:
-        all_data.append(gpt4_values)
-        all_data.append(gpt35_values)
-
     # all_data.append(baseline)
     apply_metadata(all_data)  # Populate model type based on known hardcoded values in `metadata.py`