add open-source

.gitignore

@@ -7,7 +7,6 @@ __pycache__/
 .vscode/
 
 eval-queue/
-eval-results/
 eval-queue-bk/
 eval-results-bk/
 logs/

app.py

@@ -35,27 +35,27 @@ def restart_space():
 ### Space initialisation
 try:
     print(EVAL_REQUESTS_PATH)
-    snapshot_download(
-        repo_id=QUEUE_REPO, local_dir=EVAL_REQUESTS_PATH, repo_type="dataset", tqdm_class=None, etag_timeout=30, token=TOKEN
-    )
+    # snapshot_download(
+    #     repo_id=QUEUE_REPO, local_dir=EVAL_REQUESTS_PATH, repo_type="dataset", tqdm_class=None, etag_timeout=30, token=TOKEN
+    # )
 except Exception:
     restart_space()
 try:
     print(EVAL_RESULTS_PATH)
-    snapshot_download(
-        repo_id=RESULTS_REPO, local_dir=EVAL_RESULTS_PATH, repo_type="dataset", tqdm_class=None, etag_timeout=30, token=TOKEN
-    )
+    # snapshot_download(
+    #     repo_id=RESULTS_REPO, local_dir=EVAL_RESULTS_PATH, repo_type="dataset", tqdm_class=None, etag_timeout=30, token=TOKEN
+    # )
 except Exception:
     restart_space()
 
 
 LEADERBOARD_DF = get_leaderboard_df(EVAL_RESULTS_PATH, EVAL_REQUESTS_PATH, COLS, BENCHMARK_COLS)
 
-(
-    finished_eval_queue_df,
-    running_eval_queue_df,
-    pending_eval_queue_df,
-) = get_evaluation_queue_df(EVAL_REQUESTS_PATH, EVAL_COLS)
+# (
+#     finished_eval_queue_df,
+#     running_eval_queue_df,
+#     pending_eval_queue_df,
+# ) = get_evaluation_queue_df(EVAL_REQUESTS_PATH, EVAL_COLS)
 
 def init_leaderboard(dataframe):
     if dataframe is None or dataframe.empty:
@@ -68,21 +68,21 @@ def init_leaderboard(dataframe):
             cant_deselect=[c.name for c in fields(AutoEvalColumn) if c.never_hidden],
             label="Select Columns to Display:",
         ),
-        search_columns=[AutoEvalColumn.model.name, AutoEvalColumn.license.name],
+        search_columns=[AutoEvalColumn.generation_model.name, AutoEvalColumn.retrieval_model.name],
         hide_columns=[c.name for c in fields(AutoEvalColumn) if c.hidden],
         filter_columns=[
             ColumnFilter(AutoEvalColumn.model_type.name, type="checkboxgroup", label="Model types"),
-            ColumnFilter(AutoEvalColumn.precision.name, type="checkboxgroup", label="Precision"),
-            ColumnFilter(
-                AutoEvalColumn.params.name,
-                type="slider",
-                min=0.01,
-                max=150,
-                label="Select the number of parameters (B)",
-            ),
-            ColumnFilter(
-                AutoEvalColumn.still_on_hub.name, type="boolean", label="Deleted/incomplete", default=True
-            ),
+            # ColumnFilter(AutoEvalColumn.precision.name, type="checkboxgroup", label="Precision"),
+            # ColumnFilter(
+            #     AutoEvalColumn.params.name,
+            #     type="slider",
+            #     min=0.01,
+            #     max=150,
+            #     label="Select the number of parameters (B)",
+            # ),
+            # ColumnFilter(
+            #     AutoEvalColumn.still_on_hub.name, type="boolean", label="Deleted/incomplete", default=True
+            # ),
         ],
         bool_checkboxgroup_label="Hide models",
         interactive=False,
@@ -101,92 +101,90 @@ with demo:
         with gr.TabItem("📝 About", elem_id="llm-benchmark-tab-table", id=2):
             gr.Markdown(LLM_BENCHMARKS_TEXT, elem_classes="markdown-text")
 
-        with gr.TabItem("🚀 Submit here! ", elem_id="llm-benchmark-tab-table", id=3):
-            with gr.Column():
-                with gr.Row():
-                    gr.Markdown(EVALUATION_QUEUE_TEXT, elem_classes="markdown-text")
-
-                with gr.Column():
-                    with gr.Accordion(
-                        f"✅ Finished Evaluations ({len(finished_eval_queue_df)})",
-                        open=False,
-                    ):
-                        with gr.Row():
-                            finished_eval_table = gr.components.Dataframe(
-                                value=finished_eval_queue_df,
-                                headers=EVAL_COLS,
-                                datatype=EVAL_TYPES,
-                                row_count=5,
-                            )
-                    with gr.Accordion(
-                        f"🔄 Running Evaluation Queue ({len(running_eval_queue_df)})",
-                        open=False,
-                    ):
-                        with gr.Row():
-                            running_eval_table = gr.components.Dataframe(
-                                value=running_eval_queue_df,
-                                headers=EVAL_COLS,
-                                datatype=EVAL_TYPES,
-                                row_count=5,
-                            )
-
-                    with gr.Accordion(
-                        f"⏳ Pending Evaluation Queue ({len(pending_eval_queue_df)})",
-                        open=False,
-                    ):
-                        with gr.Row():
-                            pending_eval_table = gr.components.Dataframe(
-                                value=pending_eval_queue_df,
-                                headers=EVAL_COLS,
-                                datatype=EVAL_TYPES,
-                                row_count=5,
-                            )
-            with gr.Row():
-                gr.Markdown("# ✉️✨ Submit your model here!", elem_classes="markdown-text")
-
-            with gr.Row():
-                with gr.Column():
-                    model_name_textbox = gr.Textbox(label="Model name")
-                    revision_name_textbox = gr.Textbox(label="Revision commit", placeholder="main")
-                    model_type = gr.Dropdown(
-                        choices=[t.to_str(" : ") for t in ModelType if t != ModelType.Unknown],
-                        label="Model type",
-                        multiselect=False,
-                        value=None,
-                        interactive=True,
-                    )
-
-                with gr.Column():
-                    precision = gr.Dropdown(
-                        choices=[i.value.name for i in Precision if i != Precision.Unknown],
-                        label="Precision",
-                        multiselect=False,
-                        value="float16",
-                        interactive=True,
-                    )
-                    weight_type = gr.Dropdown(
-                        choices=[i.value.name for i in WeightType],
-                        label="Weights type",
-                        multiselect=False,
-                        value="Original",
-                        interactive=True,
-                    )
-                    base_model_name_textbox = gr.Textbox(label="Base model (for delta or adapter weights)")
-
-            submit_button = gr.Button("Submit Eval")
-            submission_result = gr.Markdown()
-            submit_button.click(
-                add_new_eval,
-                [
-                    model_name_textbox,
-                    base_model_name_textbox,
-                    revision_name_textbox,
-                    precision,
-                    weight_type,
-                    model_type,
-                ],
-                submission_result,
-            )
+        # with gr.TabItem("🚀 Submit here! ", elem_id="llm-benchmark-tab-table", id=3):
+        #     with gr.Column():
+        #         with gr.Row():
+        #             gr.Markdown(EVALUATION_QUEUE_TEXT, elem_classes="markdown-text")
+        #
+        #         with gr.Column():
+        #             with gr.Accordion(
+        #                 f"✅ Finished Evaluations ({len(finished_eval_queue_df)})",
+        #                 open=False,
+        #             ):
+        #                 with gr.Row():
+        #                     finished_eval_table = gr.components.Dataframe(
+        #                         value=finished_eval_queue_df,
+        #                         headers=EVAL_COLS,
+        #                         datatype=EVAL_TYPES,
+        #                         row_count=5,
+        #                     )
+        #             with gr.Accordion(
+        #                 f"🔄 Running Evaluation Queue ({len(running_eval_queue_df)})",
+        #                 open=False,
+        #             ):
+        #                 with gr.Row():
+        #                     running_eval_table = gr.components.Dataframe(
+        #                         value=running_eval_queue_df,
+        #                         headers=EVAL_COLS,
+        #                         datatype=EVAL_TYPES,
+        #                         row_count=5,
+        #                     )
+        #
+        #             with gr.Accordion(
+        #                 f"⏳ Pending Evaluation Queue ({len(pending_eval_queue_df)})",
+        #                 open=False,
+        #             ):
+        #                 with gr.Row():
+        #                     pending_eval_table = gr.components.Dataframe(
+        #                         value=pending_eval_queue_df,
+        #                         headers=EVAL_COLS,
+        #                         datatype=EVAL_TYPES,
+        #                         row_count=5,
+        #                     )
+        #     with gr.Row():
+        #         gr.Markdown("# ✉️✨ Submit your model here!", elem_classes="markdown-text")
+        #
+        #     with gr.Row():
+        #         with gr.Column():
+        #             model_name_textbox = gr.Textbox(label="Model name")
+        #             revision_name_textbox = gr.Textbox(label="Revision commit", placeholder="main")
+        #             model_type = gr.Dropdown(
+        #                 choices=[t.to_str(" : ") for t in ModelType if t != ModelType.Unknown],
+        #                 label="Model type",
+        #                 multiselect=False,
+        #                 value=None,
+        #                 interactive=True,
+        #             )
+        #
+        #         with gr.Column():
+        #             precision = gr.Dropdown(
+        #                 choices=[i.value.name for i in Precision if i != Precision.Unknown],
+        #                 label="Precision",
+        #                 multiselect=False,
+        #                 value="float16",
+        #                 interactive=True,
+        #             )
+        #             weight_type = gr.Dropdown(
+        #                 choices=[i.value.name for i in WeightType],
+        #                 label="Weights type",
+        #                 multiselect=False,
+        #                 value="Original",
+        #                 interactive=True,
+        #             )
+        #             base_model_name_textbox = gr.Textbox(label="Base model (for delta or adapter weights)")
+        #             eval_result=gr.Textbox(label="Eval Result")
+        #
+        #     submit_button = gr.Button("Submit Eval")
+        #     submission_result = gr.Markdown()
+        #     submit_button.click(
+        #         add_new_eval,
+        #         [
+        #             model_name_textbox,
+        #             revision_name_textbox,
+        #             model_type,
+        #         ],
+        #         submission_result,
+        #     )
 
     with gr.Row():
         with gr.Accordion("📙 Citation", open=False):

eval-results/.gitattributes

@@ -0,0 +1,55 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.lz4 filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tar filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text
+# Audio files - uncompressed
+*.pcm filter=lfs diff=lfs merge=lfs -text
+*.sam filter=lfs diff=lfs merge=lfs -text
+*.raw filter=lfs diff=lfs merge=lfs -text
+# Audio files - compressed
+*.aac filter=lfs diff=lfs merge=lfs -text
+*.flac filter=lfs diff=lfs merge=lfs -text
+*.mp3 filter=lfs diff=lfs merge=lfs -text
+*.ogg filter=lfs diff=lfs merge=lfs -text
+*.wav filter=lfs diff=lfs merge=lfs -text
+# Image files - uncompressed
+*.bmp filter=lfs diff=lfs merge=lfs -text
+*.gif filter=lfs diff=lfs merge=lfs -text
+*.png filter=lfs diff=lfs merge=lfs -text
+*.tiff filter=lfs diff=lfs merge=lfs -text
+# Image files - compressed
+*.jpg filter=lfs diff=lfs merge=lfs -text
+*.jpeg filter=lfs diff=lfs merge=lfs -text
+*.webp filter=lfs diff=lfs merge=lfs -text

eval-results/demo-leaderboard/CLOSE_deepseek-v2-chat/results_2023-12-08 15:46:20.425378.json

@@ -0,0 +1,34 @@
+{
+  "results": {
+    "retrieval": {
+      "mrr": 0.0,
+      "map": 0.0
+    },
+    "generation": {
+      "em": 0.002277904328018223,
+      "f1": 0.43525331778147214,
+      "rouge1": 0.3150681120081669,
+      "rouge2": 0.12933954114035873,
+      "rougeL": 0.22495384062408755,
+      "accuracy": 0.33058086560364464,
+      "completeness": 0.5540647198105761,
+      "hallucination": 0.0,
+      "utilization": 0.0,
+      "numerical_accuracy": 0.11534391534391535
+    }
+  },
+  "config": {
+    "eval_name": "CLOSE_deepseek-v2-chat",
+    "generation_model": "deepseek-ai/DeepSeek-V2-Chat-0628",
+    "generation_model_args": {
+      "name": "deepseek-ai/DeepSeek-V2-Chat-0628",
+      "num_params": 80,
+      "open_source": true
+    },
+    "retrieval_model": "CLOSE",
+    "retrieval_model_args": {
+      "num_params": 0.0,
+      "open_source": false
+    }
+  }
+}

eval-results/demo-leaderboard/CLOSE_llama3-70b-instruct/results_2023-12-08 15:46:20.425378.json

@@ -0,0 +1,34 @@
+{
+  "results": {
+    "retrieval": {
+      "mrr": 0.0,
+      "map": 0.0
+    },
+    "generation": {
+      "em": 0.0005694760820045558,
+      "f1": 0.418257314239393,
+      "rouge1": 0.3061411048446855,
+      "rouge2": 0.12053616693649026,
+      "rougeL": 0.21948810430155005,
+      "accuracy": 0.285876993166287,
+      "completeness": 0.5132605304212169,
+      "hallucination": 0.0,
+      "utilization": 0.0,
+      "numerical_accuracy": 0.06589958158995816
+    }
+  },
+  "config": {
+    "eval_name": "CLOSE_llama3-70b-instruct",
+    "generation_model": "meta-llama/Meta-Llama-3.1-70B-Instruct",
+    "generation_model_args": {
+      "name": "meta-llama/Meta-Llama-3.1-70B-Instruct",
+      "num_params": 70,
+      "open_source": true
+    },
+    "retrieval_model": "CLOSE",
+    "retrieval_model_args": {
+      "num_params": 0.0,
+      "open_source": false
+    }
+  }
+}

eval-results/demo-leaderboard/CLOSE_qwen2-72b/results_2023-12-08 15:46:20.425378.json

@@ -0,0 +1,34 @@
+{
+  "results": {
+    "retrieval": {
+      "mrr": 0.0,
+      "map": 0.0
+    },
+    "generation": {
+      "em": 0.0005694760820045558,
+      "f1": 0.4191216882279184,
+      "rouge1": 0.2989940495432677,
+      "rouge2": 0.12047626678426614,
+      "rougeL": 0.2082230205185154,
+      "accuracy": 0.34054669703872437,
+      "completeness": 0.5753690753690753,
+      "hallucination": 0.0,
+      "utilization": 0.0,
+      "numerical_accuracy": 0.12406417112299466
+    }
+  },
+  "config": {
+    "eval_name": "CLOSE_qwen2-72b",
+    "generation_model": "Qwen/Qwen2.5-72B-Instruct",
+    "generation_model_args": {
+      "name": "Qwen/Qwen2.5-72B-Instruct",
+      "num_params": 72,
+      "open_source": true
+    },
+    "retrieval_model": "CLOSE",
+    "retrieval_model_args": {
+      "num_params": 0.0,
+      "open_source": false
+    }
+  }
+}

eval-results/demo-leaderboard/CLOSE_yi15-34b/results_2023-12-08 15:46:20.425378.json

@@ -0,0 +1,34 @@
+{
+  "results": {
+    "retrieval": {
+      "mrr": 0.0,
+      "map": 0.0
+    },
+    "generation": {
+      "em": 0.0,
+      "f1": 0.11613328937628616,
+      "rouge1": 0.15613267640197273,
+      "rouge2": 0.04591153663411247,
+      "rougeL": 0.0496843687172552,
+      "accuracy": 0.14607061503416857,
+      "completeness": 0.4987157534246575,
+      "hallucination": 0.0,
+      "utilization": 0.0,
+      "numerical_accuracy": 0.0748663101604278
+    }
+  },
+  "config": {
+    "eval_name": "CLOSE_yi15-34b",
+    "generation_model": "01ai/Yi-1.5-34B-Chat-16K",
+    "generation_model_args": {
+      "name": "01ai/Yi-1.5-34B-Chat-16K",
+      "num_params": 34,
+      "open_source": true
+    },
+    "retrieval_model": "CLOSE",
+    "retrieval_model_args": {
+      "num_params": 0.0,
+      "open_source": false
+    }
+  }
+}

eval-results/demo-leaderboard/gte-qwen2-1.5b_deepseek-v2-chat/results_2023-12-08 15:46:20.425378.json

@@ -0,0 +1,35 @@
+{
+  "results": {
+    "retrieval": {
+      "mrr": 0.36173120728929387,
+      "map": 0.3512338648443432
+    },
+    "generation": {
+      "em": 0.0056947608200455585,
+      "f1": 0.4212862409737785,
+      "rouge1": 0.3707328288930376,
+      "rouge2": 0.21393113234607009,
+      "rougeL": 0.2719847145278759,
+      "accuracy": 0.3886674259681093,
+      "completeness": 0.5858823529411765,
+      "hallucination": 0.07893209518282066,
+      "utilization": 0.48166472642607683,
+      "numerical_accuracy": 0.27365491651205937
+    }
+  },
+  "config": {
+    "eval_name": "gte-qwen2-1.5b_deepseek-v2-chat",
+    "generation_model": "deepseek-ai/DeepSeek-V2-Chat-0628",
+    "generation_model_args": {
+      "name": "deepseek-ai/DeepSeek-V2-Chat-0628",
+      "num_params": 80,
+      "open_source": true
+    },
+    "retrieval_model": "Alibaba-NLP/gte-Qwen2-1.5B-instruct",
+    "retrieval_model_args": {
+      "name": "Alibaba-NLP/gte-Qwen2-1.5B-instruct",
+      "num_params": 1.5,
+      "open_source": true
+    }
+  }
+}

eval-results/demo-leaderboard/gte-qwen2-1.5b_llama3-70b-instruct/results_2023-12-08 15:46:20.425378.json

@@ -0,0 +1,35 @@
+{
+  "results": {
+    "retrieval": {
+      "mrr": 0.36173120728929387,
+      "map": 0.3512338648443432
+    },
+    "generation": {
+      "em": 0.04555808656036447,
+      "f1": 0.4907954247383474,
+      "rouge1": 0.4080491070348775,
+      "rouge2": 0.23130474174425783,
+      "rougeL": 0.3217574785678875,
+      "accuracy": 0.4216970387243736,
+      "completeness": 0.5688146380270486,
+      "hallucination": 0.11832946635730858,
+      "utilization": 0.4491869918699187,
+      "numerical_accuracy": 0.288981288981289
+    }
+  },
+  "config": {
+    "eval_name": "gte-qwen2-1.5b_llama3-70b-instruct",
+    "generation_model": "meta-llama/Meta-Llama-3.1-70B-Instruct",
+    "generation_model_args": {
+      "name": "meta-llama/Meta-Llama-3.1-70B-Instruct",
+      "num_params": 70,
+      "open_source": true
+    },
+    "retrieval_model": "Alibaba-NLP/gte-Qwen2-1.5B-instruct",
+    "retrieval_model_args": {
+      "name": "Alibaba-NLP/gte-Qwen2-1.5B-instruct",
+      "num_params": 1.5,
+      "open_source": true
+    }
+  }
+}

eval-results/demo-leaderboard/gte-qwen2-1.5b_qwen2-72b/results_2023-12-08 15:46:20.425378.json

@@ -0,0 +1,35 @@
+{
+  "results": {
+    "retrieval": {
+      "mrr": 0.36173120728929387,
+      "map": 0.3512338648443432
+    },
+    "generation": {
+      "em": 0.002277904328018223,
+      "f1": 0.3804001391052641,
+      "rouge1": 0.34576336184459094,
+      "rouge2": 0.1928778762677512,
+      "rougeL": 0.2383694455084706,
+      "accuracy": 0.4145785876993166,
+      "completeness": 0.598297213622291,
+      "hallucination": 0.07213496218731821,
+      "utilization": 1.13922942206655,
+      "numerical_accuracy": 0.3218694885361552
+    }
+  },
+  "config": {
+    "eval_name": "gte-qwen2-1.5b_qwen2-72b",
+    "generation_model": "Qwen/Qwen2.5-72B-Instruct",
+    "generation_model_args": {
+      "name": "Qwen/Qwen2.5-72B-Instruct",
+      "num_params": 72,
+      "open_source": true
+    },
+    "retrieval_model": "Alibaba-NLP/gte-Qwen2-1.5B-instruct",
+    "retrieval_model_args": {
+      "name": "Alibaba-NLP/gte-Qwen2-1.5B-instruct",
+      "num_params": 1.5,
+      "open_source": true
+    }
+  }
+}

eval-results/demo-leaderboard/gte-qwen2-1.5b_yi15-34b/results_2023-12-08 15:46:20.425378.json

@@ -0,0 +1,35 @@
+{
+  "results": {
+    "retrieval": {
+      "mrr": 0.36173120728929387,
+      "map": 0.3512338648443432
+    },
+    "generation": {
+      "em": 0.0,
+      "f1": 0.16041349053275844,
+      "rouge1": 0.21775697114621573,
+      "rouge2": 0.09738983880706074,
+      "rougeL": 0.08775246194460379,
+      "accuracy": 0.3211845102505695,
+      "completeness": 0.5703789636504254,
+      "hallucination": 0.07665094339622641,
+      "utilization": 0.40828402366863903,
+      "numerical_accuracy": 0.162
+    }
+  },
+  "config": {
+    "eval_name": "gte-qwen2-1.5b_yi15-34b",
+    "generation_model": "01ai/Yi-1.5-34B-Chat-16K",
+    "generation_model_args": {
+      "name": "01ai/Yi-1.5-34B-Chat-16K",
+      "num_params": 34,
+      "open_source": true
+    },
+    "retrieval_model": "Alibaba-NLP/gte-Qwen2-1.5B-instruct",
+    "retrieval_model_args": {
+      "name": "Alibaba-NLP/gte-Qwen2-1.5B-instruct",
+      "num_params": 1.5,
+      "open_source": true
+    }
+  }
+}

eval-results/demo-leaderboard/qwen2-72b_bge-large-zh/results_2023-12-08 15:46:20.425378.json

@@ -0,0 +1,35 @@
+{
+  "results": {
+    "retrieval": {
+      "mrr": 0.3426063022019742,
+      "map": 0.33500379650721335
+    },
+    "generation": {
+      "em": 0.0017084282460136675,
+      "f1": 0.3797528411547138,
+      "rouge1": 0.3372893350582966,
+      "rouge2": 0.18329984910669803,
+      "rougeL": 0.23230144566069125,
+      "accuracy": 0.40888382687927105,
+      "completeness": 0.6021044427123928,
+      "hallucination": 0.0023391812865497076,
+      "utilization": 0.5014637002341921,
+      "numerical_accuracy": 0.3100358422939068
+    }
+  },
+  "config": {
+    "eval_name": "bge-large-zh_bge-large-zh",
+    "generation_model": "BAAI/bge-large-zh",
+    "generation_model_args": {
+      "name": "BAAI/bge-large-zh",
+      "num_params": 0.2,
+      "open_source": true
+    },
+    "retrieval_model": "BAAI/bge-large-zh",
+    "retrieval_model_args": {
+      "name": "BAAI/bge-large-zh",
+      "num_params": 0.2,
+      "open_source": true
+    }
+  }
+}

eval-results/demo-leaderboard/qwen2-72b_bge-m3/results_2023-12-08 15:46:20.425378.json

@@ -0,0 +1,35 @@
+{
+  "results": {
+    "retrieval": {
+      "mrr": 0.3527809415337889,
+      "map": 0.3458855353075171
+    },
+    "generation": {
+      "em": 0.0017084282460136675,
+      "f1": 0.38645032979631466,
+      "rouge1": 0.3467267951634575,
+      "rouge2": 0.1930581604826183,
+      "rougeL": 0.24141093461883717,
+      "accuracy": 0.4271070615034169,
+      "completeness": 0.6119287374128582,
+      "hallucination": 0.0005847953216374269,
+      "utilization": 0.5400116822429907,
+      "numerical_accuracy": 0.3372093023255814
+    }
+  },
+  "config": {
+    "eval_name": "bge-m3_bge-m3",
+    "generation_model": "BAAI/bge-m3",
+    "generation_model_args": {
+      "name": "BAAI/bge-m3",
+      "num_params": 0.2,
+      "open_source": true
+    },
+    "retrieval_model": "BAAI/bge-m3",
+    "retrieval_model_args": {
+      "name": "BAAI/bge-m3",
+      "num_params": 0.2,
+      "open_source": true
+    }
+  }
+}

eval-results/demo-leaderboard/qwen2-72b_e5-mistral-7b/results_2023-12-08 15:46:20.425378.json

@@ -0,0 +1,35 @@
+{
+  "results": {
+    "retrieval": {
+      "mrr": 0.303246013667426,
+      "map": 0.2960516324981017
+    },
+    "generation": {
+      "em": 0.002277904328018223,
+      "f1": 0.3705164550873997,
+      "rouge1": 0.3270311806826159,
+      "rouge2": 0.17476659877087528,
+      "rougeL": 0.22225645997479143,
+      "accuracy": 0.385250569476082,
+      "completeness": 0.5877535101404057,
+      "hallucination": 1.2922719349215572,
+      "utilization": 0.4793244030285381,
+      "numerical_accuracy": 0.28622540250447226
+    }
+  },
+  "config": {
+    "eval_name": "e5-mistral-7b_e5-mistral-7b",
+    "generation_model": "intfloat/e5-mistral-7b-instruct",
+    "generation_model_args": {
+      "name": "intfloat/e5-mistral-7b-instruct",
+      "num_params": 7,
+      "open_source": true
+    },
+    "retrieval_model": "intfloat/e5-mistral-7b-instruct",
+    "retrieval_model_args": {
+      "name": "intfloat/e5-mistral-7b-instruct",
+      "num_params": 7,
+      "open_source": true
+    }
+  }
+}

eval-results/demo-leaderboard/qwen2-72b_gte-qwen2-1.5b/results_2023-12-08 15:46:20.425378.json

@@ -0,0 +1,35 @@
+{
+  "results": {
+    "retrieval": {
+      "mrr": 0.36173120728929387,
+      "map": 0.3512338648443432
+    },
+    "generation": {
+      "em": 0.002277904328018223,
+      "f1": 0.3804001391052641,
+      "rouge1": 0.34576336184459094,
+      "rouge2": 0.1928778762677512,
+      "rougeL": 0.2383694455084706,
+      "accuracy": 0.4145785876993166,
+      "completeness": 0.598297213622291,
+      "hallucination": 0.0011627906976744186,
+      "utilization": 1.13922942206655,
+      "numerical_accuracy": 0.3218694885361552
+    }
+  },
+  "config": {
+    "eval_name": "gte-qwen2-1.5b_gte-qwen2-1.5b",
+    "generation_model": "Alibaba-NLP/gte-Qwen2-1.5B-instruct",
+    "generation_model_args": {
+      "name": "Alibaba-NLP/gte-Qwen2-1.5B-instruct",
+      "num_params": 1.5,
+      "open_source": true
+    },
+    "retrieval_model": "Alibaba-NLP/gte-Qwen2-1.5B-instruct",
+    "retrieval_model_args": {
+      "name": "Alibaba-NLP/gte-Qwen2-1.5B-instruct",
+      "num_params": 1.5,
+      "open_source": true
+    }
+  }
+}

eval-results/demo-leaderboard/qwen2-72b_jina-zh/results_2023-12-08 15:46:20.425378.json

@@ -0,0 +1,35 @@
+{
+  "results": {
+    "retrieval": {
+      "mrr": 0.27484813971146543,
+      "map": 0.26924354593773725
+    },
+    "generation": {
+      "em": 0.003416856492027335,
+      "f1": 0.37960439080933656,
+      "rouge1": 0.3255380867320351,
+      "rouge2": 0.1732248556904568,
+      "rougeL": 0.22591939162851002,
+      "accuracy": 0.3826879271070615,
+      "completeness": 0.5793588741204065,
+      "hallucination": 0.0017381228273464658,
+      "utilization": 0.4855072463768116,
+      "numerical_accuracy": 0.2663594470046083
+    }
+  },
+  "config": {
+    "eval_name": "jina-zh_jina-zh",
+    "generation_model": "jinaai/reader-lm-0.5b",
+    "generation_model_args": {
+      "name": "jinaai/reader-lm-0.5b",
+      "num_params": 0.2,
+      "open_source": true
+    },
+    "retrieval_model": "jinaai/reader-lm-0.5b",
+    "retrieval_model_args": {
+      "name": "jinaai/reader-lm-0.5b",
+      "num_params": 0.2,
+      "open_source": true
+    }
+  }
+}

src/about.py

@@ -12,8 +12,26 @@ class Task:
 # ---------------------------------------------------
 class Tasks(Enum):
     # task_key in the json file, metric_key in the json file, name to display in the leaderboard 
-    task0 = Task("anli_r1", "acc", "ANLI")
-    task1 = Task("logiqa", "acc_norm", "LogiQA")
+    # task0 = Task("anli_r1", "acc", "ANLI")
+    # task1 = Task("logiqa", "acc_norm", "LogiQA")
+
+    # retrieval tasks
+    mrr = Task("retrieval", "mrr", "MRR")
+    map = Task("retrieval", "map", "MAP")
+
+    # generation tasks
+    em = Task("generation", "em", "EM")
+    f1 = Task("generation", "f1", "F1")
+    rouge1 = Task("generation", "rouge1", "Rouge-1")
+    rouge2 = Task("generation", "rouge2", "Rouge-2")
+    rougeL = Task("generation", "rougeL", "Rouge-L")
+
+    accuracy = Task("generation", "accuracy", "ACC")
+    completeness = Task("generation", "completeness", "COMP")
+    hallucination = Task("generation", "hallucination", "HAL")
+    utilization = Task("generation", "utilization", "UTIL")
+    numerical_accuracy = Task("generation", "numerical_accuracy", "MACC")
+
 
 NUM_FEWSHOT = 0 # Change with your few shot
 # ---------------------------------------------------
@@ -21,7 +39,7 @@ NUM_FEWSHOT = 0 # Change with your few shot
 
 
 # Your leaderboard name
-TITLE = """<h1 align="center" id="space-title">Demo leaderboard</h1>"""
+TITLE = """<h1 align="center" id="space-title">Fin Benchmark leaderboard</h1>"""
 
 # What does your leaderboard evaluate?
 INTRODUCTION_TEXT = """

src/display/formatting.py

@@ -2,7 +2,9 @@ def model_hyperlink(link, model_name):
     return f'<a target="_blank" href="{link}" style="color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;">{model_name}</a>'
 
 
-def make_clickable_model(model_name):
+def make_clickable_model(model_name, model_link=None):
+    if model_link:
+        return model_hyperlink(model_link, model_name)
     link = f"https://huggingface.co/{model_name}"
     return model_hyperlink(link, model_name)
 

src/display/utils.py

@@ -1,8 +1,6 @@
 from dataclasses import dataclass, make_dataclass
 from enum import Enum
 
-import pandas as pd
-
 from src.about import Tasks
 
 def fields(raw_class):
@@ -23,35 +21,42 @@ class ColumnContent:
 ## Leaderboard columns
 auto_eval_column_dict = []
 # Init
-auto_eval_column_dict.append(["model_type_symbol", ColumnContent, ColumnContent("T", "str", True, never_hidden=True)])
-auto_eval_column_dict.append(["model", ColumnContent, ColumnContent("Model", "markdown", True, never_hidden=True)])
+auto_eval_column_dict.append(["model_type_symbol", ColumnContent, ColumnContent("Type Symbol", "str", True, never_hidden=True)])
+# auto_eval_column_dict.append(["model", ColumnContent, ColumnContent("Model", "markdown", True, never_hidden=True)])
+auto_eval_column_dict.append(["retrieval_model", ColumnContent, ColumnContent("Retrieval Model", "markdown", True, never_hidden=True)])
+auto_eval_column_dict.append(["generation_model", ColumnContent, ColumnContent("Generation Model", "markdown", True, never_hidden=True)])
+
 #Scores
 auto_eval_column_dict.append(["average", ColumnContent, ColumnContent("Average ⬆️", "number", True)])
 for task in Tasks:
     auto_eval_column_dict.append([task.name, ColumnContent, ColumnContent(task.value.col_name, "number", True)])
+
 # Model information
 auto_eval_column_dict.append(["model_type", ColumnContent, ColumnContent("Type", "str", False)])
-auto_eval_column_dict.append(["architecture", ColumnContent, ColumnContent("Architecture", "str", False)])
-auto_eval_column_dict.append(["weight_type", ColumnContent, ColumnContent("Weight type", "str", False, True)])
-auto_eval_column_dict.append(["precision", ColumnContent, ColumnContent("Precision", "str", False)])
-auto_eval_column_dict.append(["license", ColumnContent, ColumnContent("Hub License", "str", False)])
-auto_eval_column_dict.append(["params", ColumnContent, ColumnContent("#Params (B)", "number", False)])
-auto_eval_column_dict.append(["likes", ColumnContent, ColumnContent("Hub ❤️", "number", False)])
-auto_eval_column_dict.append(["still_on_hub", ColumnContent, ColumnContent("Available on the hub", "bool", False)])
-auto_eval_column_dict.append(["revision", ColumnContent, ColumnContent("Model sha", "str", False, False)])
+# auto_eval_column_dict.append(["architecture", ColumnContent, ColumnContent("Architecture", "str", False)])
+# auto_eval_column_dict.append(["weight_type", ColumnContent, ColumnContent("Weight type", "str", False, True)])
+# auto_eval_column_dict.append(["precision", ColumnContent, ColumnContent("Precision", "str", False)])
+# auto_eval_column_dict.append(["license", ColumnContent, ColumnContent("Hub License", "str", False)])
+auto_eval_column_dict.append(["gen_num_params", ColumnContent, ColumnContent("Gen#Params (B)", "number", False)])
+auto_eval_column_dict.append(["ret_num_params", ColumnContent, ColumnContent("Ret#Params (B)", "number", False)])
+# auto_eval_column_dict.append(["likes", ColumnContent, ColumnContent("Hub ❤️", "number", False)])
+# auto_eval_column_dict.append(["still_on_hub", ColumnContent, ColumnContent("Available on the hub", "bool", False)])
+# auto_eval_column_dict.append(["revision", ColumnContent, ColumnContent("Model sha", "str", False, False)])
 
 # We use make dataclass to dynamically fill the scores from Tasks
 AutoEvalColumn = make_dataclass("AutoEvalColumn", auto_eval_column_dict, frozen=True)
+EvalQueueColumn = make_dataclass("EvalQueueColumn", auto_eval_column_dict, frozen=True)
 
 ## For the queue columns in the submission tab
-@dataclass(frozen=True)
-class EvalQueueColumn:  # Queue column
-    model = ColumnContent("model", "markdown", True)
-    revision = ColumnContent("revision", "str", True)
-    private = ColumnContent("private", "bool", True)
-    precision = ColumnContent("precision", "str", True)
-    weight_type = ColumnContent("weight_type", "str", "Original")
-    status = ColumnContent("status", "str", True)
+# @dataclass(frozen=True)
+# class EvalQueueColumn:  # Queue column
+#     model = ColumnContent("model", "markdown", True)
+    # revision = ColumnContent("revision", "str", True)
+    # private = ColumnContent("private", "bool", True)
+    # precision = ColumnContent("precision", "str", True)
+    # weight_type = ColumnContent("weight_type", "str", "Original")
+    # status = ColumnContent("status", "str", True)
+
 
 ## All the model information that we might need
 @dataclass
@@ -62,10 +67,10 @@ class ModelDetails:
 
 
 class ModelType(Enum):
-    PT = ModelDetails(name="pretrained", symbol="🟢")
-    FT = ModelDetails(name="fine-tuned", symbol="🔶")
-    IFT = ModelDetails(name="instruction-tuned", symbol="⭕")
-    RL = ModelDetails(name="RL-tuned", symbol="🟦")
+    OpenSource = ModelDetails(name="open-source", symbol="🟢")
+    # FT = ModelDetails(name="fine-tuned", symbol="🔶")
+    ClosedSource = ModelDetails(name="closed-source", symbol="⭕")
+    # RL = ModelDetails(name="RL-tuned", symbol="🟦")
     Unknown = ModelDetails(name="", symbol="?")
 
     def to_str(self, separator=" "):
@@ -73,14 +78,10 @@ class ModelType(Enum):
 
     @staticmethod
     def from_str(type):
-        if "fine-tuned" in type or "🔶" in type:
-            return ModelType.FT
-        if "pretrained" in type or "🟢" in type:
-            return ModelType.PT
-        if "RL-tuned" in type or "🟦" in type:
-            return ModelType.RL
-        if "instruction-tuned" in type or "⭕" in type:
-            return ModelType.IFT
+        if "open-source" in type or "🟢" in type:
+            return ModelType.OpenSource
+        if "closed-source" in type or "⭕" in type:
+            return ModelType.ClosedSource
         return ModelType.Unknown
 
 class WeightType(Enum):

src/leaderboard/read_evals.py

@@ -17,18 +17,21 @@ class EvalResult:
     """Represents one full evaluation. Built from a combination of the result and request file for a given run.
     """
     eval_name: str # org_model_precision (uid)
-    full_model: str # org/model (path on hub)
     org: str 
-    model: str
-    revision: str # commit hash, "" if main
+    generation_model: str
+    retrieval_model: str
+    # revision: str # commit hash, "" if main
     results: dict
+    generation_model_link: str = "" # link to the model on the hub
+    generation_model_args: dict = None
+    retrieval_model_link: str = "" # link to the model on the hub
+    retrieval_model_args: dict = None
     precision: Precision = Precision.Unknown
     model_type: ModelType = ModelType.Unknown # Pretrained, fine tuned, ...
     weight_type: WeightType = WeightType.Original # Original or Adapter
     architecture: str = "Unknown" 
     license: str = "?"
     likes: int = 0
-    num_params: int = 0
     date: str = "" # submission date of request file
     still_on_hub: bool = False
 
@@ -41,30 +44,32 @@ class EvalResult:
         config = data.get("config")
 
         # Precision
-        precision = Precision.from_str(config.get("model_dtype"))
+        # precision = Precision.from_str(config.get("model_dtype"))
 
         # Get model and org
-        org_and_model = config.get("model_name", config.get("model_args", None))
-        org_and_model = org_and_model.split("/", 1)
-
-        if len(org_and_model) == 1:
-            org = None
-            model = org_and_model[0]
-            result_key = f"{model}_{precision.value.name}"
-        else:
-            org = org_and_model[0]
-            model = org_and_model[1]
-            result_key = f"{org}_{model}_{precision.value.name}"
-        full_model = "/".join(org_and_model)
-
-        still_on_hub, _, model_config = is_model_on_hub(
-            full_model, config.get("model_sha", "main"), trust_remote_code=True, test_tokenizer=False
-        )
-        architecture = "?"
-        if model_config is not None:
-            architectures = getattr(model_config, "architectures", None)
-            if architectures:
-                architecture = ";".join(architectures)
+        eval_name= config.get("eval_name", "")
+        generation_model = config.get("generation_model", "")
+        retrieval_model = config.get("retrieval_model", "")
+        org= config.get("org", "")
+        # org_and_model = org_and_model.split("/", 1)
+        #
+        # if len(org_and_model) == 1:
+        #     org = None
+        #     model = org_and_model[0]
+        #     result_key = f"{model}_{precision.value.name}"
+        # else:
+        #     org = org_and_model[0]
+        #     model = org_and_model[1]
+        #     result_key = f"{org}_{model}_{precision.value.name}"
+        # full_model = "/".join(org_and_model)
+
+        # still_on_hub, _, model_config = is_model_on_hub(
+        #     full_model, config.get("model_sha", "main"), trust_remote_code=True, test_tokenizer=False
+        # )
+        # if model_config is not None:
+        #     architectures = getattr(model_config, "architectures", None)
+        #     if architectures:
+        #         architecture = ";".join(architectures)
 
         # Extract results available in this file (some results are split in several files)
         results = {}
@@ -79,16 +84,28 @@ class EvalResult:
             mean_acc = np.mean(accs) * 100.0
             results[task.benchmark] = mean_acc
 
+        generation_model_args = config.get("generation_model_args", None)
+        retrieval_model_args = config.get("retrieval_model_args", None)
+        open_source= True
+        if not generation_model_args or not generation_model_args.get("open_source", False):
+            open_source = False
+        if not retrieval_model_args or not retrieval_model_args.get("open_source", False):
+            open_source = False
+
         return self(
-            eval_name=result_key,
-            full_model=full_model,
+            eval_name=eval_name,
+            # full_model=full_model,
             org=org,
-            model=model,
+            generation_model=generation_model,
+            retrieval_model=retrieval_model,
             results=results,
-            precision=precision,  
-            revision= config.get("model_sha", ""),
-            still_on_hub=still_on_hub,
-            architecture=architecture
+            generation_model_args=generation_model_args,
+            retrieval_model_args=retrieval_model_args,
+            model_type=ModelType.OpenSource if open_source else ModelType.ClosedSource,
+            # precision=precision,
+            # revision= config.get("model_sha", ""),
+            # still_on_hub=still_on_hub,
+            # architecture=architecture
         )
 
     def update_with_request_file(self, requests_path):
@@ -112,18 +129,21 @@ class EvalResult:
         average = sum([v for v in self.results.values() if v is not None]) / len(Tasks)
         data_dict = {
             "eval_name": self.eval_name,  # not a column, just a save name,
-            AutoEvalColumn.precision.name: self.precision.value.name,
+            # AutoEvalColumn.precision.name: self.precision.value.name,
             AutoEvalColumn.model_type.name: self.model_type.value.name,
             AutoEvalColumn.model_type_symbol.name: self.model_type.value.symbol,
-            AutoEvalColumn.weight_type.name: self.weight_type.value.name,
-            AutoEvalColumn.architecture.name: self.architecture,
-            AutoEvalColumn.model.name: make_clickable_model(self.full_model),
-            AutoEvalColumn.revision.name: self.revision,
+            # AutoEvalColumn.weight_type.name: self.weight_type.value.name,
+            # AutoEvalColumn.architecture.name: self.architecture,
+            AutoEvalColumn.generation_model.name: make_clickable_model(self.generation_model, self.generation_model_link),
+            AutoEvalColumn.retrieval_model.name: make_clickable_model(self.retrieval_model, self.retrieval_model_link),
+            # AutoEvalColumn.revision.name: self.revision,
             AutoEvalColumn.average.name: average,
-            AutoEvalColumn.license.name: self.license,
-            AutoEvalColumn.likes.name: self.likes,
-            AutoEvalColumn.params.name: self.num_params,
-            AutoEvalColumn.still_on_hub.name: self.still_on_hub,
+            # AutoEvalColumn.license.name: self.license,
+            # AutoEvalColumn.likes.name: self.likes,
+            # AutoEvalColumn.generation_model_params.name: self.num_params,
+            # AutoEvalColumn.still_on_hub.name: self.still_on_hub,
+            "Gen#Params (B)": self.generation_model_args.get("num_params", "Unknown"),
+            "Ret#Params (B)": self.retrieval_model_args.get("num_params", "Unknown"),
         }
 
         for task in Tasks:
@@ -171,12 +191,13 @@ def get_raw_eval_results(results_path: str, requests_path: str) -> list[EvalResu
 
         for file in files:
             model_result_filepaths.append(os.path.join(root, file))
+            print(f"Adding {file}")
 
     eval_results = {}
     for model_result_filepath in model_result_filepaths:
         # Creation of result
         eval_result = EvalResult.init_from_json_file(model_result_filepath)
-        eval_result.update_with_request_file(requests_path)
+        # eval_result.update_with_request_file(requests_path)
 
         # Store results of same eval together
         eval_name = eval_result.eval_name
@@ -191,6 +212,8 @@ def get_raw_eval_results(results_path: str, requests_path: str) -> list[EvalResu
             v.to_dict() # we test if the dict version is complete
             results.append(v)
         except KeyError:  # not all eval values present
+            import traceback
+            traceback.print_exc()
             continue
 
     return results

src/populate.py

@@ -12,6 +12,7 @@ def get_leaderboard_df(results_path: str, requests_path: str, cols: list, benchm
     """Creates a dataframe from all the individual experiment results"""
     raw_data = get_raw_eval_results(results_path, requests_path)
     all_data_json = [v.to_dict() for v in raw_data]
+    print(all_data_json)
 
     df = pd.DataFrame.from_records(all_data_json)
     df = df.sort_values(by=[AutoEvalColumn.average.name], ascending=False)