Update app.py

app.py

@@ -16,32 +16,6 @@ import plotly.graph_objects as go
 CATEGORIES = ["task-solving", "math-reasoning", "general-instruction", "natural-question", "safety"]
 LANGS = ['en', 'vi', 'th', 'id', 'km', 'lo', 'ms', 'my', 'tl']
 
-# benchmark_name = "sea_bench_all"
-
-# with open(f"data/{benchmark_name}/question.jsonl", 'r') as f:
-#     questions = [
-#         json.loads(x)
-#         for x in f
-#     ]
-#     questions = {
-#         q['question_id']: q
-#         for q in questions
-#     }
-
-
-# def get_model_df():
-#     cnt = 0
-#     q2result = []
-#     fin = open(f"data/{benchmark_name}/model_judgment/gpt-4_single.jsonl", "r")
-#     for line in fin:
-#         obj = json.loads(line)
-#         # obj["category"] = CATEGORIES[(obj["question_id"]-81)//10]
-#         obj["category"] = questions[obj['question_id']]['category']
-#         obj["lang"] = questions[obj['question_id']]['lang']
-#         q2result.append(obj)
-#     df = pd.DataFrame(q2result)
-#     return df
-
 
 force_download = bool(int(os.environ.get("force_download", "1")))
 HF_TOKEN = str(os.environ.get("HF_TOKEN", ""))
@@ -50,6 +24,97 @@ PERFORMANCE_FILENAME = str(os.environ.get("PERFORMANCE_FILENAME", "gpt4_single_j
 
 MODEL_DFRAME = None
 
+
+CATEGORIES = ["task-solving", "math-reasoning", "general-instruction", "natural-question", "safety"]
+LANGS = ['en', 'vi', 'th', 'id', 'km', 'lo', 'ms', 'my', 'tl']
+
+FORCE_DOWNLOAD = bool(int(os.environ.get("FORCE_DOWNLOAD", "0")))
+HF_TOKEN = str(os.environ.get("HF_TOKEN", ""))
+DATA_SET_REPO_PATH = str(os.environ.get("DATA_SET_REPO_PATH", "SeaLLMs/Sea-bench"))
+
+PERFORMANCE_FILENAME = str(os.environ.get("PERFORMANCE_FILENAME", "model_judgment/gpt-4_single.jsonl"))
+QUESTION_FILE_NAME = str(os.environ.get("QUESTION_FILE_NAME", "question.jsonl"))
+
+rename_map = {
+    "seallm-13b-chat": "SeaLLM-13b",
+    "polylm-13b": "PolyLM-13b",
+    "qwen-14b": "Qwen-14b",
+    "gpt-3.5-turbo": "GPT-3.5-turbo",
+}
+CATEGORIES = [ "task-solving",  "math-reasoning",  "general-instruction",  "natural-question",  "safety", ]
+
+CATEGORIES_NAMES = {
+    "task-solving": 'Task-solving', 
+    "math-reasoning": 'Math', 
+    "general-instruction": 'General-instruction', 
+    "natural-question": 'NaturalQA', 
+    "safety": 'Safety',
+}
+
+LANGS = ['en', 'vi', 'id', 'ms', 'tl', 'th', 'km', 'lo', 'my']
+LANG_NAMES = {
+    'en': 'eng', 
+    'vi': 'vie', 
+    'th': 'tha', 
+    'id': 'ind', 
+    'km': 'khm', 
+    'lo': 'lao', 
+    'ms': 'msa', 
+    'my': 'mya', 
+    'tl': 'tgl',
+}
+
+
+
+MODEL_DFRAME = None
+
+
+def read_jsonl_report(question_path, file_path):
+    with open(question_path, 'r') as f:
+        questions = [
+            json.loads(x)
+            for x in f
+        ]
+        questions = {
+            q['question_id']: q
+            for q in questions
+        }
+
+    q2result = []
+    fin = open(file_path, "r")
+    for line in fin:
+        obj = json.loads(line)
+        obj["category"] = questions[obj['question_id']]['category']
+        obj["lang"] = questions[obj['question_id']]['lang']
+        q2result.append(obj)
+    df = pd.DataFrame(q2result)
+    return df
+
+def get_report_df_from_jsonl():
+    from huggingface_hub import hf_hub_download
+    assert DATA_SET_REPO_PATH != ''
+    assert HF_TOKEN != ''
+    repo_id = DATA_SET_REPO_PATH
+    question_path = hf_hub_download(
+        repo_id=repo_id,
+        filename=QUESTION_FILE_NAME,
+        force_download=FORCE_DOWNLOAD,
+        local_dir='./hf_cache',
+        repo_type="dataset",
+        token=HF_TOKEN
+    )
+    file_path = hf_hub_download(
+        repo_id=repo_id,
+        filename=PERFORMANCE_FILENAME,
+        force_download=FORCE_DOWNLOAD,
+        local_dir='./hf_cache',
+        repo_type="dataset",
+        token=HF_TOKEN
+    )
+    print(f'Downloaded file at {question_path}/ {file_path} from {DATA_SET_REPO_PATH} / {PERFORMANCE_FILENAME}')
+    return read_jsonl_report(question_path, file_path)
+
+
 def get_model_df():
     global MODEL_DFRAME
     if isinstance(MODEL_DFRAME, pd.DataFrame):
@@ -121,43 +186,9 @@ def polar_subplot(fig, dframe, model_names, category_label, category_names, row,
         )
         fig.add_trace(polar, row, col)
 
-rename_map = {
-    # "seallm13b10L4k_a_sft4xdpo_5a": "SeaLLM-13b-10L",
-    "seallm13b10L6k_a_5a1R1_seaall_sft4x_1_5a1_r2_0_dpo_8_40000s": "SeaLLM-13b",
-    "polylm": "PolyLM-13b",
-    "qwen": "Qwen-14b",
-    "gpt-3.5-turbo": "GPT-3.5-turbo",
-    "gpt-4-1106-preview": "GPT-4-turbo",
-}
-CATEGORIES = [ "task-solving",  "math-reasoning",  "general-instruction",  "natural-question",  "safety", ]
-
-CATEGORIES_NAMES = {
-    "task-solving": 'Task-solving', 
-    "math-reasoning": 'Math', 
-    "general-instruction": 'General-instruction', 
-    "natural-question": 'NaturalQA', 
-    "safety": 'Safety',
-}
-
-
-# LANGS = ['en', 'vi', 'th', 'id', 'km', 'lo', 'ms', 'my', 'tl']
-LANGS = ['en', 'vi', 'id', 'ms', 'tl', 'th', 'km', 'lo', 'my']
-LANG_NAMES = {
-    'en': 'eng', 
-    'vi': 'vie', 
-    'th': 'tha', 
-    'id': 'ind', 
-    'km': 'khm', 
-    'lo': 'lao', 
-    'ms': 'msa', 
-    'my': 'mya', 
-    'tl': 'tgl',
-
-}
-
 
 def plot_agg_fn():
-    df = get_model_df()
+    df = get_report_df_from_jsonl()
 
     all_models = df["model"].unique()
     model_names = list(rename_map.items())
@@ -228,7 +259,7 @@ def plot_agg_fn():
 
 
 def plot_by_lang_fn():
-    df = get_model_df()
+    df = get_report_df_from_jsonl()
     model_names = list(rename_map.items())
 
     fig = make_subplots(