Update space

.idea/.gitignore

@@ -0,0 +1,8 @@
+# Default ignored files
+/shelf/
+/workspace.xml
+# Editor-based HTTP Client requests
+/httpRequests/
+# Datasource local storage ignored files
+/dataSources/
+/dataSources.local.xml

.idea/Leaderboard.iml

@@ -0,0 +1,12 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<module type="PYTHON_MODULE" version="4">
+  <component name="NewModuleRootManager">
+    <content url="file://$MODULE_DIR$" />
+    <orderEntry type="inheritedJdk" />
+    <orderEntry type="sourceFolder" forTests="false" />
+  </component>
+  <component name="PyDocumentationSettings">
+    <option name="format" value="PLAIN" />
+    <option name="myDocStringFormat" value="Plain" />
+  </component>
+</module>

.idea/inspectionProfiles/Project_Default.xml

@@ -0,0 +1,12 @@
+<component name="InspectionProjectProfileManager">
+  <profile version="1.0">
+    <option name="myName" value="Project Default" />
+    <inspection_tool class="PyUnresolvedReferencesInspection" enabled="true" level="WARNING" enabled_by_default="true">
+      <option name="ignoredIdentifiers">
+        <list>
+          <option value="db_file" />
+        </list>
+      </option>
+    </inspection_tool>
+  </profile>
+</component>

.idea/inspectionProfiles/profiles_settings.xml

@@ -0,0 +1,6 @@
+<component name="InspectionProjectProfileManager">
+  <settings>
+    <option name="USE_PROJECT_PROFILE" value="false" />
+    <version value="1.0" />
+  </settings>
+</component>

.idea/misc.xml

@@ -0,0 +1,7 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="Black">
+    <option name="sdkName" value="leaderboard" />
+  </component>
+  <component name="ProjectRootManager" version="2" project-jdk-name="leaderboard" project-jdk-type="Python SDK" />
+</project>

.idea/modules.xml

@@ -0,0 +1,8 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="ProjectModuleManager">
+    <modules>
+      <module fileurl="file://$PROJECT_DIR$/.idea/Leaderboard.iml" filepath="$PROJECT_DIR$/.idea/Leaderboard.iml" />
+    </modules>
+  </component>
+</project>

.idea/vcs.xml

@@ -0,0 +1,6 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="VcsDirectoryMappings">
+    <mapping directory="" vcs="Git" />
+  </component>
+</project>

all_dimensions/Audience.jsonl

@@ -0,0 +1,10 @@
+{"Rank": 1, "Model": "<a target=\"_blank\" style=\"text-decoration: underline\" href=\"https://huggingface.co/\">Model_7</a>", "WISE": 0.99, "SICR": 0.94, "nDCG@10(Original)": 0.89, "nDCG@10(Instructed)": 0.53, "nDCG@10(Reversely Instructed)": 0.97, "MRR@1(Original)": 0.68, "MRR@1(Instructed)": 0.92, "MRR@1(Reversely Instructed)": 0.09}
+{"Rank": 2, "Model": "<a target=\"_blank\" style=\"text-decoration: underline\" href=\"https://huggingface.co/\">Model_9</a>", "WISE": 0.84, "SICR": 0.21, "nDCG@10(Original)": 0.5, "nDCG@10(Instructed)": 0.68, "nDCG@10(Reversely Instructed)": 0.86, "MRR@1(Original)": 0.79, "MRR@1(Instructed)": 0.91, "MRR@1(Reversely Instructed)": 0.12}
+{"Rank": 3, "Model": "<a target=\"_blank\" style=\"text-decoration: underline\" href=\"https://huggingface.co/\">Model_6</a>", "WISE": 0.71, "SICR": 0.49, "nDCG@10(Original)": 0.74, "nDCG@10(Instructed)": 0.35, "nDCG@10(Reversely Instructed)": 0.56, "MRR@1(Original)": 0.34, "MRR@1(Instructed)": 0.92, "MRR@1(Reversely Instructed)": 0.27}
+{"Rank": 4, "Model": "<a target=\"_blank\" style=\"text-decoration: underline\" href=\"https://huggingface.co/\">Model_3</a>", "WISE": 0.52, "SICR": 0.99, "nDCG@10(Original)": 0.08, "nDCG@10(Instructed)": 0.2, "nDCG@10(Reversely Instructed)": 0.55, "MRR@1(Original)": 0.09, "MRR@1(Instructed)": 0.58, "MRR@1(Reversely Instructed)": 0.59}
+{"Rank": 5, "Model": "<a target=\"_blank\" style=\"text-decoration: underline\" href=\"https://huggingface.co/\">Model_8</a>", "WISE": 0.39, "SICR": 0.61, "nDCG@10(Original)": 0.21, "nDCG@10(Instructed)": 0.46, "nDCG@10(Reversely Instructed)": 0.13, "MRR@1(Original)": 0.35, "MRR@1(Instructed)": 0.8, "MRR@1(Reversely Instructed)": 0.1}
+{"Rank": 6, "Model": "<a target=\"_blank\" style=\"text-decoration: underline\" href=\"https://huggingface.co/\">Model_2</a>", "WISE": 0.35, "SICR": 0.82, "nDCG@10(Original)": 0.42, "nDCG@10(Instructed)": 0.92, "nDCG@10(Reversely Instructed)": 0.17, "MRR@1(Original)": 0.39, "MRR@1(Instructed)": 0.24, "MRR@1(Reversely Instructed)": 0.32}
+{"Rank": 7, "Model": "<a target=\"_blank\" style=\"text-decoration: underline\" href=\"https://huggingface.co/\">Model_1</a>", "WISE": 0.26, "SICR": 0.17, "nDCG@10(Original)": 0.68, "nDCG@10(Instructed)": 0.73, "nDCG@10(Reversely Instructed)": 0.58, "MRR@1(Original)": 0.13, "MRR@1(Instructed)": 0.62, "MRR@1(Reversely Instructed)": 0.9}
+{"Rank": 8, "Model": "<a target=\"_blank\" style=\"text-decoration: underline\" href=\"https://huggingface.co/\">Model_4</a>", "WISE": 0.22, "SICR": 0.69, "nDCG@10(Original)": 0.44, "nDCG@10(Instructed)": 0.25, "nDCG@10(Reversely Instructed)": 0.99, "MRR@1(Original)": 0.41, "MRR@1(Instructed)": 0.35, "MRR@1(Reversely Instructed)": 0.16}
+{"Rank": 9, "Model": "<a target=\"_blank\" style=\"text-decoration: underline\" href=\"https://huggingface.co/\">Model_0</a>", "WISE": 0.13, "SICR": 0.13, "nDCG@10(Original)": 0.71, "nDCG@10(Instructed)": 0.85, "nDCG@10(Reversely Instructed)": 0.09, "MRR@1(Original)": 0.08, "MRR@1(Instructed)": 0.92, "MRR@1(Reversely Instructed)": 0.09}
+{"Rank": 10, "Model": "<a target=\"_blank\" style=\"text-decoration: underline\" href=\"https://huggingface.co/\">Model_5</a>", "WISE": 0.01, "SICR": 0.65, "nDCG@10(Original)": 0.01, "nDCG@10(Instructed)": 0.35, "nDCG@10(Reversely Instructed)": 0.82, "MRR@1(Original)": 0.69, "MRR@1(Instructed)": 0.94, "MRR@1(Reversely Instructed)": 0.96}

all_dimensions/Format.jsonl

@@ -0,0 +1,10 @@
+{"Rank": 1, "Model": "<a target=\"_blank\" style=\"text-decoration: underline\" href=\"https://huggingface.co/\">Model_4</a>", "WISE": 0.92, "SICR": 0.33, "nDCG@10(Original)": 0.02, "nDCG@10(Instructed)": 0.23, "nDCG@10(Reversely Instructed)": 0.65, "MRR@1(Original)": 0.1, "MRR@1(Instructed)": 0.1, "MRR@1(Reversely Instructed)": 0.51}
+{"Rank": 2, "Model": "<a target=\"_blank\" style=\"text-decoration: underline\" href=\"https://huggingface.co/\">Model_0</a>", "WISE": 0.88, "SICR": 0.35, "nDCG@10(Original)": 0.69, "nDCG@10(Instructed)": 0.48, "nDCG@10(Reversely Instructed)": 0.25, "MRR@1(Original)": 0.3, "MRR@1(Instructed)": 0.92, "MRR@1(Reversely Instructed)": 0.03}
+{"Rank": 3, "Model": "<a target=\"_blank\" style=\"text-decoration: underline\" href=\"https://huggingface.co/\">Model_8</a>", "WISE": 0.81, "SICR": 0.32, "nDCG@10(Original)": 0.22, "nDCG@10(Instructed)": 0.67, "nDCG@10(Reversely Instructed)": 0.79, "MRR@1(Original)": 0.59, "MRR@1(Instructed)": 0.97, "MRR@1(Reversely Instructed)": 0.05}
+{"Rank": 4, "Model": "<a target=\"_blank\" style=\"text-decoration: underline\" href=\"https://huggingface.co/\">Model_1</a>", "WISE": 0.7, "SICR": 0.69, "nDCG@10(Original)": 0.82, "nDCG@10(Instructed)": 0.31, "nDCG@10(Reversely Instructed)": 0.08, "MRR@1(Original)": 0.57, "MRR@1(Instructed)": 0.54, "MRR@1(Reversely Instructed)": 0.34}
+{"Rank": 5, "Model": "<a target=\"_blank\" style=\"text-decoration: underline\" href=\"https://huggingface.co/\">Model_7</a>", "WISE": 0.67, "SICR": 0.04, "nDCG@10(Original)": 0.61, "nDCG@10(Instructed)": 0.28, "nDCG@10(Reversely Instructed)": 0.62, "MRR@1(Original)": 0.22, "MRR@1(Instructed)": 0.85, "MRR@1(Reversely Instructed)": 0.09}
+{"Rank": 6, "Model": "<a target=\"_blank\" style=\"text-decoration: underline\" href=\"https://huggingface.co/\">Model_2</a>", "WISE": 0.56, "SICR": 0.86, "nDCG@10(Original)": 0.98, "nDCG@10(Instructed)": 0.22, "nDCG@10(Reversely Instructed)": 0.95, "MRR@1(Original)": 0.69, "MRR@1(Instructed)": 0.63, "MRR@1(Reversely Instructed)": 0.22}
+{"Rank": 7, "Model": "<a target=\"_blank\" style=\"text-decoration: underline\" href=\"https://huggingface.co/\">Model_6</a>", "WISE": 0.33, "SICR": 0.92, "nDCG@10(Original)": 0.38, "nDCG@10(Instructed)": 0.31, "nDCG@10(Reversely Instructed)": 0.09, "MRR@1(Original)": 0.26, "MRR@1(Instructed)": 0.89, "MRR@1(Reversely Instructed)": 0.79}
+{"Rank": 8, "Model": "<a target=\"_blank\" style=\"text-decoration: underline\" href=\"https://huggingface.co/\">Model_3</a>", "WISE": 0.21, "SICR": 0.82, "nDCG@10(Original)": 0.85, "nDCG@10(Instructed)": 0.97, "nDCG@10(Reversely Instructed)": 0.69, "MRR@1(Original)": 0.81, "MRR@1(Instructed)": 0.49, "MRR@1(Reversely Instructed)": 0.43}
+{"Rank": 9, "Model": "<a target=\"_blank\" style=\"text-decoration: underline\" href=\"https://huggingface.co/\">Model_9</a>", "WISE": 0.15, "SICR": 0.94, "nDCG@10(Original)": 0.02, "nDCG@10(Instructed)": 0.9, "nDCG@10(Reversely Instructed)": 0.95, "MRR@1(Original)": 0.89, "MRR@1(Instructed)": 0.32, "MRR@1(Reversely Instructed)": 0.12}
+{"Rank": 10, "Model": "<a target=\"_blank\" style=\"text-decoration: underline\" href=\"https://huggingface.co/\">Model_5</a>", "WISE": 0.11, "SICR": 0.06, "nDCG@10(Original)": 0.09, "nDCG@10(Instructed)": 0.72, "nDCG@10(Reversely Instructed)": 0.61, "MRR@1(Original)": 0.16, "MRR@1(Instructed)": 0.29, "MRR@1(Reversely Instructed)": 0.61}

all_dimensions/Keyword.jsonl

@@ -0,0 +1,10 @@
+{"Rank": 1, "Model": "<a target=\"_blank\" style=\"text-decoration: underline\" href=\"https://huggingface.co/\">Model_9</a>", "WISE": 0.83, "SICR": 0.8, "nDCG@10(Original)": 0.61, "nDCG@10(Instructed)": 0.19, "nDCG@10(Reversely Instructed)": 0.46, "MRR@1(Original)": 0.13, "MRR@1(Instructed)": 0.37, "MRR@1(Reversely Instructed)": 0.97}
+{"Rank": 2, "Model": "<a target=\"_blank\" style=\"text-decoration: underline\" href=\"https://huggingface.co/\">Model_4</a>", "WISE": 0.78, "SICR": 0.56, "nDCG@10(Original)": 0.73, "nDCG@10(Instructed)": 0.49, "nDCG@10(Reversely Instructed)": 0.34, "MRR@1(Original)": 0.18, "MRR@1(Instructed)": 0.2, "MRR@1(Reversely Instructed)": 0.52}
+{"Rank": 3, "Model": "<a target=\"_blank\" style=\"text-decoration: underline\" href=\"https://huggingface.co/\">Model_6</a>", "WISE": 0.76, "SICR": 0.12, "nDCG@10(Original)": 0.05, "nDCG@10(Instructed)": 0.24, "nDCG@10(Reversely Instructed)": 0.89, "MRR@1(Original)": 0.03, "MRR@1(Instructed)": 0.99, "MRR@1(Reversely Instructed)": 0.45}
+{"Rank": 4, "Model": "<a target=\"_blank\" style=\"text-decoration: underline\" href=\"https://huggingface.co/\">Model_2</a>", "WISE": 0.73, "SICR": 0.35, "nDCG@10(Original)": 0.31, "nDCG@10(Instructed)": 0.18, "nDCG@10(Reversely Instructed)": 0.93, "MRR@1(Original)": 0.01, "MRR@1(Instructed)": 0.26, "MRR@1(Reversely Instructed)": 0.73}
+{"Rank": 5, "Model": "<a target=\"_blank\" style=\"text-decoration: underline\" href=\"https://huggingface.co/\">Model_3</a>", "WISE": 0.62, "SICR": 0.58, "nDCG@10(Original)": 0.66, "nDCG@10(Instructed)": 0.58, "nDCG@10(Reversely Instructed)": 0.62, "MRR@1(Original)": 0.64, "MRR@1(Instructed)": 0.12, "MRR@1(Reversely Instructed)": 0.68}
+{"Rank": 6, "Model": "<a target=\"_blank\" style=\"text-decoration: underline\" href=\"https://huggingface.co/\">Model_7</a>", "WISE": 0.54, "SICR": 0.42, "nDCG@10(Original)": 0.34, "nDCG@10(Instructed)": 0.03, "nDCG@10(Reversely Instructed)": 0.88, "MRR@1(Original)": 0.99, "MRR@1(Instructed)": 0.27, "MRR@1(Reversely Instructed)": 0.37}
+{"Rank": 7, "Model": "<a target=\"_blank\" style=\"text-decoration: underline\" href=\"https://huggingface.co/\">Model_8</a>", "WISE": 0.47, "SICR": 0.43, "nDCG@10(Original)": 0.4, "nDCG@10(Instructed)": 0.61, "nDCG@10(Reversely Instructed)": 0.49, "MRR@1(Original)": 0.23, "MRR@1(Instructed)": 0.94, "MRR@1(Reversely Instructed)": 0.41}
+{"Rank": 8, "Model": "<a target=\"_blank\" style=\"text-decoration: underline\" href=\"https://huggingface.co/\">Model_0</a>", "WISE": 0.34, "SICR": 0.49, "nDCG@10(Original)": 0.15, "nDCG@10(Instructed)": 0.98, "nDCG@10(Reversely Instructed)": 0.9, "MRR@1(Original)": 0.77, "MRR@1(Instructed)": 0.29, "MRR@1(Reversely Instructed)": 0.73}
+{"Rank": 9, "Model": "<a target=\"_blank\" style=\"text-decoration: underline\" href=\"https://huggingface.co/\">Model_1</a>", "WISE": 0.14, "SICR": 0.92, "nDCG@10(Original)": 0.39, "nDCG@10(Instructed)": 0.1, "nDCG@10(Reversely Instructed)": 0.78, "MRR@1(Original)": 0.35, "MRR@1(Instructed)": 0.36, "MRR@1(Reversely Instructed)": 1.0}
+{"Rank": 10, "Model": "<a target=\"_blank\" style=\"text-decoration: underline\" href=\"https://huggingface.co/\">Model_5</a>", "WISE": 0.06, "SICR": 0.16, "nDCG@10(Original)": 0.4, "nDCG@10(Instructed)": 0.44, "nDCG@10(Reversely Instructed)": 0.53, "MRR@1(Original)": 0.22, "MRR@1(Instructed)": 0.64, "MRR@1(Reversely Instructed)": 0.14}

all_dimensions/Language.jsonl

@@ -0,0 +1,10 @@
+{"Rank": 1, "Model": "<a target=\"_blank\" style=\"text-decoration: underline\" href=\"https://huggingface.co/\">Model_4</a>", "WISE": 0.74, "SICR": 0.41, "nDCG@10(Original)": 0.97, "nDCG@10(Instructed)": 0.43, "nDCG@10(Reversely Instructed)": 0.2, "MRR@1(Original)": 0.45, "MRR@1(Instructed)": 0.41, "MRR@1(Reversely Instructed)": 0.88}
+{"Rank": 2, "Model": "<a target=\"_blank\" style=\"text-decoration: underline\" href=\"https://huggingface.co/\">Model_8</a>", "WISE": 0.65, "SICR": 0.88, "nDCG@10(Original)": 0.75, "nDCG@10(Instructed)": 0.14, "nDCG@10(Reversely Instructed)": 0.43, "MRR@1(Original)": 0.05, "MRR@1(Instructed)": 0.75, "MRR@1(Reversely Instructed)": 0.12}
+{"Rank": 3, "Model": "<a target=\"_blank\" style=\"text-decoration: underline\" href=\"https://huggingface.co/\">Model_9</a>", "WISE": 0.65, "SICR": 0.25, "nDCG@10(Original)": 0.95, "nDCG@10(Instructed)": 0.14, "nDCG@10(Reversely Instructed)": 0.94, "MRR@1(Original)": 0.81, "MRR@1(Instructed)": 0.37, "MRR@1(Reversely Instructed)": 0.28}
+{"Rank": 4, "Model": "<a target=\"_blank\" style=\"text-decoration: underline\" href=\"https://huggingface.co/\">Model_7</a>", "WISE": 0.48, "SICR": 0.38, "nDCG@10(Original)": 0.37, "nDCG@10(Instructed)": 0.32, "nDCG@10(Reversely Instructed)": 0.29, "MRR@1(Original)": 0.26, "MRR@1(Instructed)": 0.51, "MRR@1(Reversely Instructed)": 0.84}
+{"Rank": 5, "Model": "<a target=\"_blank\" style=\"text-decoration: underline\" href=\"https://huggingface.co/\">Model_1</a>", "WISE": 0.24, "SICR": 0.26, "nDCG@10(Original)": 0.55, "nDCG@10(Instructed)": 0.89, "nDCG@10(Reversely Instructed)": 0.62, "MRR@1(Original)": 0.73, "MRR@1(Instructed)": 0.84, "MRR@1(Reversely Instructed)": 0.79}
+{"Rank": 6, "Model": "<a target=\"_blank\" style=\"text-decoration: underline\" href=\"https://huggingface.co/\">Model_3</a>", "WISE": 0.22, "SICR": 0.77, "nDCG@10(Original)": 0.71, "nDCG@10(Instructed)": 0.53, "nDCG@10(Reversely Instructed)": 0.21, "MRR@1(Original)": 0.22, "MRR@1(Instructed)": 0.82, "MRR@1(Reversely Instructed)": 0.2}
+{"Rank": 7, "Model": "<a target=\"_blank\" style=\"text-decoration: underline\" href=\"https://huggingface.co/\">Model_0</a>", "WISE": 0.18, "SICR": 0.25, "nDCG@10(Original)": 0.53, "nDCG@10(Instructed)": 0.64, "nDCG@10(Reversely Instructed)": 0.58, "MRR@1(Original)": 0.32, "MRR@1(Instructed)": 0.41, "MRR@1(Reversely Instructed)": 0.15}
+{"Rank": 8, "Model": "<a target=\"_blank\" style=\"text-decoration: underline\" href=\"https://huggingface.co/\">Model_2</a>", "WISE": 0.09, "SICR": 0.43, "nDCG@10(Original)": 0.82, "nDCG@10(Instructed)": 0.78, "nDCG@10(Reversely Instructed)": 0.43, "MRR@1(Original)": 0.03, "MRR@1(Instructed)": 0.53, "MRR@1(Reversely Instructed)": 0.83}
+{"Rank": 9, "Model": "<a target=\"_blank\" style=\"text-decoration: underline\" href=\"https://huggingface.co/\">Model_5</a>", "WISE": 0.07, "SICR": 0.14, "nDCG@10(Original)": 0.17, "nDCG@10(Instructed)": 0.86, "nDCG@10(Reversely Instructed)": 0.9, "MRR@1(Original)": 0.93, "MRR@1(Instructed)": 0.99, "MRR@1(Reversely Instructed)": 0.96}
+{"Rank": 10, "Model": "<a target=\"_blank\" style=\"text-decoration: underline\" href=\"https://huggingface.co/\">Model_6</a>", "WISE": 0.05, "SICR": 0.29, "nDCG@10(Original)": 0.45, "nDCG@10(Instructed)": 0.74, "nDCG@10(Reversely Instructed)": 0.93, "MRR@1(Original)": 0.23, "MRR@1(Instructed)": 0.65, "MRR@1(Reversely Instructed)": 0.46}

all_dimensions/Length.jsonl

@@ -0,0 +1,10 @@
+{"Rank": 1, "Model": "<a target=\"_blank\" style=\"text-decoration: underline\" href=\"https://huggingface.co/\">Model_2</a>", "WISE": 0.98, "SICR": 0.77, "nDCG@10(Original)": 0.28, "nDCG@10(Instructed)": 0.09, "nDCG@10(Reversely Instructed)": 0.32, "MRR@1(Original)": 0.25, "MRR@1(Instructed)": 0.22, "MRR@1(Reversely Instructed)": 0.81}
+{"Rank": 2, "Model": "<a target=\"_blank\" style=\"text-decoration: underline\" href=\"https://huggingface.co/\">Model_0</a>", "WISE": 0.97, "SICR": 0.51, "nDCG@10(Original)": 0.55, "nDCG@10(Instructed)": 0.41, "nDCG@10(Reversely Instructed)": 0.4, "MRR@1(Original)": 0.21, "MRR@1(Instructed)": 0.14, "MRR@1(Reversely Instructed)": 0.9}
+{"Rank": 3, "Model": "<a target=\"_blank\" style=\"text-decoration: underline\" href=\"https://huggingface.co/\">Model_7</a>", "WISE": 0.95, "SICR": 0.8, "nDCG@10(Original)": 0.72, "nDCG@10(Instructed)": 0.32, "nDCG@10(Reversely Instructed)": 0.08, "MRR@1(Original)": 0.89, "MRR@1(Instructed)": 0.05, "MRR@1(Reversely Instructed)": 0.3}
+{"Rank": 4, "Model": "<a target=\"_blank\" style=\"text-decoration: underline\" href=\"https://huggingface.co/\">Model_6</a>", "WISE": 0.89, "SICR": 0.35, "nDCG@10(Original)": 0.65, "nDCG@10(Instructed)": 0.19, "nDCG@10(Reversely Instructed)": 0.88, "MRR@1(Original)": 0.06, "MRR@1(Instructed)": 0.9, "MRR@1(Reversely Instructed)": 0.02}
+{"Rank": 5, "Model": "<a target=\"_blank\" style=\"text-decoration: underline\" href=\"https://huggingface.co/\">Model_8</a>", "WISE": 0.78, "SICR": 0.18, "nDCG@10(Original)": 0.28, "nDCG@10(Instructed)": 0.89, "nDCG@10(Reversely Instructed)": 0.34, "MRR@1(Original)": 0.8, "MRR@1(Instructed)": 0.49, "MRR@1(Reversely Instructed)": 0.01}
+{"Rank": 6, "Model": "<a target=\"_blank\" style=\"text-decoration: underline\" href=\"https://huggingface.co/\">Model_4</a>", "WISE": 0.77, "SICR": 0.46, "nDCG@10(Original)": 0.98, "nDCG@10(Instructed)": 0.99, "nDCG@10(Reversely Instructed)": 0.19, "MRR@1(Original)": 0.77, "MRR@1(Instructed)": 0.11, "MRR@1(Reversely Instructed)": 0.36}
+{"Rank": 7, "Model": "<a target=\"_blank\" style=\"text-decoration: underline\" href=\"https://huggingface.co/\">Model_3</a>", "WISE": 0.75, "SICR": 0.88, "nDCG@10(Original)": 0.07, "nDCG@10(Instructed)": 0.75, "nDCG@10(Reversely Instructed)": 0.78, "MRR@1(Original)": 0.57, "MRR@1(Instructed)": 0.12, "MRR@1(Reversely Instructed)": 0.38}
+{"Rank": 8, "Model": "<a target=\"_blank\" style=\"text-decoration: underline\" href=\"https://huggingface.co/\">Model_5</a>", "WISE": 0.61, "SICR": 0.27, "nDCG@10(Original)": 0.1, "nDCG@10(Instructed)": 0.11, "nDCG@10(Reversely Instructed)": 0.78, "MRR@1(Original)": 0.3, "MRR@1(Instructed)": 0.96, "MRR@1(Reversely Instructed)": 0.58}
+{"Rank": 9, "Model": "<a target=\"_blank\" style=\"text-decoration: underline\" href=\"https://huggingface.co/\">Model_9</a>", "WISE": 0.2, "SICR": 0.73, "nDCG@10(Original)": 0.68, "nDCG@10(Instructed)": 0.01, "nDCG@10(Reversely Instructed)": 0.12, "MRR@1(Original)": 0.7, "MRR@1(Instructed)": 0.62, "MRR@1(Reversely Instructed)": 0.79}
+{"Rank": 10, "Model": "<a target=\"_blank\" style=\"text-decoration: underline\" href=\"https://huggingface.co/\">Model_1</a>", "WISE": 0.08, "SICR": 0.21, "nDCG@10(Original)": 0.92, "nDCG@10(Instructed)": 0.91, "nDCG@10(Reversely Instructed)": 0.81, "MRR@1(Original)": 0.16, "MRR@1(Instructed)": 0.01, "MRR@1(Reversely Instructed)": 0.22}

all_dimensions/Source.jsonl

@@ -0,0 +1,10 @@
+{"Rank": 1, "Model": "<a target=\"_blank\" style=\"text-decoration: underline\" href=\"https://huggingface.co/\">Model_9</a>", "WISE": 0.84, "SICR": 0.72, "nDCG@10(Original)": 0.31, "nDCG@10(Instructed)": 0.18, "nDCG@10(Reversely Instructed)": 0.75, "MRR@1(Original)": 0.48, "MRR@1(Instructed)": 0.98, "MRR@1(Reversely Instructed)": 0.86}
+{"Rank": 2, "Model": "<a target=\"_blank\" style=\"text-decoration: underline\" href=\"https://huggingface.co/\">Model_3</a>", "WISE": 0.69, "SICR": 0.62, "nDCG@10(Original)": 0.76, "nDCG@10(Instructed)": 0.53, "nDCG@10(Reversely Instructed)": 0.5, "MRR@1(Original)": 0.68, "MRR@1(Instructed)": 0.67, "MRR@1(Reversely Instructed)": 0.81}
+{"Rank": 3, "Model": "<a target=\"_blank\" style=\"text-decoration: underline\" href=\"https://huggingface.co/\">Model_0</a>", "WISE": 0.61, "SICR": 0.84, "nDCG@10(Original)": 0.55, "nDCG@10(Instructed)": 0.78, "nDCG@10(Reversely Instructed)": 0.03, "MRR@1(Original)": 0.24, "MRR@1(Instructed)": 0.59, "MRR@1(Reversely Instructed)": 0.15}
+{"Rank": 4, "Model": "<a target=\"_blank\" style=\"text-decoration: underline\" href=\"https://huggingface.co/\">Model_7</a>", "WISE": 0.6, "SICR": 0.86, "nDCG@10(Original)": 0.46, "nDCG@10(Instructed)": 0.8, "nDCG@10(Reversely Instructed)": 0.21, "MRR@1(Original)": 0.55, "MRR@1(Instructed)": 0.26, "MRR@1(Reversely Instructed)": 0.42}
+{"Rank": 5, "Model": "<a target=\"_blank\" style=\"text-decoration: underline\" href=\"https://huggingface.co/\">Model_6</a>", "WISE": 0.55, "SICR": 0.73, "nDCG@10(Original)": 0.23, "nDCG@10(Instructed)": 0.56, "nDCG@10(Reversely Instructed)": 0.47, "MRR@1(Original)": 0.89, "MRR@1(Instructed)": 0.23, "MRR@1(Reversely Instructed)": 0.83}
+{"Rank": 6, "Model": "<a target=\"_blank\" style=\"text-decoration: underline\" href=\"https://huggingface.co/\">Model_1</a>", "WISE": 0.48, "SICR": 0.1, "nDCG@10(Original)": 0.12, "nDCG@10(Instructed)": 0.56, "nDCG@10(Reversely Instructed)": 0.93, "MRR@1(Original)": 0.67, "MRR@1(Instructed)": 0.66, "MRR@1(Reversely Instructed)": 0.02}
+{"Rank": 7, "Model": "<a target=\"_blank\" style=\"text-decoration: underline\" href=\"https://huggingface.co/\">Model_4</a>", "WISE": 0.32, "SICR": 0.77, "nDCG@10(Original)": 0.76, "nDCG@10(Instructed)": 0.83, "nDCG@10(Reversely Instructed)": 0.13, "MRR@1(Original)": 0.14, "MRR@1(Instructed)": 0.0, "MRR@1(Reversely Instructed)": 0.08}
+{"Rank": 8, "Model": "<a target=\"_blank\" style=\"text-decoration: underline\" href=\"https://huggingface.co/\">Model_8</a>", "WISE": 0.29, "SICR": 0.42, "nDCG@10(Original)": 0.45, "nDCG@10(Instructed)": 0.31, "nDCG@10(Reversely Instructed)": 0.7, "MRR@1(Original)": 0.66, "MRR@1(Instructed)": 0.12, "MRR@1(Reversely Instructed)": 0.44}
+{"Rank": 9, "Model": "<a target=\"_blank\" style=\"text-decoration: underline\" href=\"https://huggingface.co/\">Model_2</a>", "WISE": 0.17, "SICR": 0.91, "nDCG@10(Original)": 0.76, "nDCG@10(Instructed)": 0.78, "nDCG@10(Reversely Instructed)": 0.7, "MRR@1(Original)": 0.77, "MRR@1(Instructed)": 0.88, "MRR@1(Reversely Instructed)": 0.55}
+{"Rank": 10, "Model": "<a target=\"_blank\" style=\"text-decoration: underline\" href=\"https://huggingface.co/\">Model_5</a>", "WISE": 0.13, "SICR": 0.44, "nDCG@10(Original)": 0.4, "nDCG@10(Instructed)": 0.93, "nDCG@10(Reversely Instructed)": 0.28, "MRR@1(Original)": 0.68, "MRR@1(Instructed)": 0.43, "MRR@1(Reversely Instructed)": 0.44}

app.py

@@ -2,201 +2,68 @@ import gradio as gr
 from gradio_leaderboard import Leaderboard, ColumnFilter, SelectColumns
 import pandas as pd
 from apscheduler.schedulers.background import BackgroundScheduler
-from huggingface_hub import snapshot_download
-
-from src.about import (
-    CITATION_BUTTON_LABEL,
-    CITATION_BUTTON_TEXT,
-    EVALUATION_QUEUE_TEXT,
-    INTRODUCTION_TEXT,
-    LLM_BENCHMARKS_TEXT,
-    TITLE,
-)
-from src.display.css_html_js import custom_css
-from src.display.utils import (
-    BENCHMARK_COLS,
-    COLS,
-    EVAL_COLS,
-    EVAL_TYPES,
-    AutoEvalColumn,
-    ModelType,
-    fields,
-    WeightType,
-    Precision
-)
+
 from src.envs import API, EVAL_REQUESTS_PATH, EVAL_RESULTS_PATH, QUEUE_REPO, REPO_ID, RESULTS_REPO, TOKEN
-from src.populate import get_evaluation_queue_df, get_leaderboard_df
-from src.submission.submit import add_new_eval
+
+from utils import get_data, submit
 
 
 def restart_space():
     API.restart_space(repo_id=REPO_ID)
 
-### Space initialisation
-try:
-    print(EVAL_REQUESTS_PATH)
-    snapshot_download(
-        repo_id=QUEUE_REPO, local_dir=EVAL_REQUESTS_PATH, repo_type="dataset", tqdm_class=None, etag_timeout=30, token=TOKEN
-    )
-except Exception:
-    restart_space()
-try:
-    print(EVAL_RESULTS_PATH)
-    snapshot_download(
-        repo_id=RESULTS_REPO, local_dir=EVAL_RESULTS_PATH, repo_type="dataset", tqdm_class=None, etag_timeout=30, token=TOKEN
-    )
-except Exception:
-    restart_space()
-
-
-LEADERBOARD_DF = get_leaderboard_df(EVAL_RESULTS_PATH, EVAL_REQUESTS_PATH, COLS, BENCHMARK_COLS)
-
-(
-    finished_eval_queue_df,
-    running_eval_queue_df,
-    pending_eval_queue_df,
-) = get_evaluation_queue_df(EVAL_REQUESTS_PATH, EVAL_COLS)
-
-def init_leaderboard(dataframe):
-    if dataframe is None or dataframe.empty:
-        raise ValueError("Leaderboard DataFrame is empty or None.")
-    return Leaderboard(
-        value=dataframe,
-        datatype=[c.type for c in fields(AutoEvalColumn)],
-        select_columns=SelectColumns(
-            default_selection=[c.name for c in fields(AutoEvalColumn) if c.displayed_by_default],
-            cant_deselect=[c.name for c in fields(AutoEvalColumn) if c.never_hidden],
-            label="Select Columns to Display:",
-        ),
-        search_columns=[AutoEvalColumn.model.name, AutoEvalColumn.license.name],
-        hide_columns=[c.name for c in fields(AutoEvalColumn) if c.hidden],
-        filter_columns=[
-            ColumnFilter(AutoEvalColumn.model_type.name, type="checkboxgroup", label="Model types"),
-            ColumnFilter(AutoEvalColumn.precision.name, type="checkboxgroup", label="Precision"),
-            ColumnFilter(
-                AutoEvalColumn.params.name,
-                type="slider",
-                min=0.01,
-                max=150,
-                label="Select the number of parameters (B)",
-            ),
-            ColumnFilter(
-                AutoEvalColumn.still_on_hub.name, type="boolean", label="Deleted/incomplete", default=True
-            ),
-        ],
-        bool_checkboxgroup_label="Hide models",
-        interactive=False,
-    )
-
-
-demo = gr.Blocks(css=custom_css)
-with demo:
-    gr.HTML(TITLE)
-    gr.Markdown(INTRODUCTION_TEXT, elem_classes="markdown-text")
-
-    with gr.Tabs(elem_classes="tab-buttons") as tabs:
-        with gr.TabItem("🏅 LLM Benchmark", elem_id="llm-benchmark-tab-table", id=0):
-            leaderboard = init_leaderboard(LEADERBOARD_DF)
-
-        with gr.TabItem("📝 About", elem_id="llm-benchmark-tab-table", id=2):
-            gr.Markdown(LLM_BENCHMARKS_TEXT, elem_classes="markdown-text")
-
-        with gr.TabItem("🚀 Submit here! ", elem_id="llm-benchmark-tab-table", id=3):
-            with gr.Column():
-                with gr.Row():
-                    gr.Markdown(EVALUATION_QUEUE_TEXT, elem_classes="markdown-text")
-
-                with gr.Column():
-                    with gr.Accordion(
-                        f"✅ Finished Evaluations ({len(finished_eval_queue_df)})",
-                        open=False,
-                    ):
-                        with gr.Row():
-                            finished_eval_table = gr.components.Dataframe(
-                                value=finished_eval_queue_df,
-                                headers=EVAL_COLS,
-                                datatype=EVAL_TYPES,
-                                row_count=5,
-                            )
-                    with gr.Accordion(
-                        f"🔄 Running Evaluation Queue ({len(running_eval_queue_df)})",
-                        open=False,
-                    ):
-                        with gr.Row():
-                            running_eval_table = gr.components.Dataframe(
-                                value=running_eval_queue_df,
-                                headers=EVAL_COLS,
-                                datatype=EVAL_TYPES,
-                                row_count=5,
-                            )
-
-                    with gr.Accordion(
-                        f"⏳ Pending Evaluation Queue ({len(pending_eval_queue_df)})",
-                        open=False,
-                    ):
-                        with gr.Row():
-                            pending_eval_table = gr.components.Dataframe(
-                                value=pending_eval_queue_df,
-                                headers=EVAL_COLS,
-                                datatype=EVAL_TYPES,
-                                row_count=5,
-                            )
-            with gr.Row():
-                gr.Markdown("# ✉️✨ Submit your model here!", elem_classes="markdown-text")
-
-            with gr.Row():
-                with gr.Column():
-                    model_name_textbox = gr.Textbox(label="Model name")
-                    revision_name_textbox = gr.Textbox(label="Revision commit", placeholder="main")
-                    model_type = gr.Dropdown(
-                        choices=[t.to_str(" : ") for t in ModelType if t != ModelType.Unknown],
-                        label="Model type",
-                        multiselect=False,
-                        value=None,
-                        interactive=True,
-                    )
-
-                with gr.Column():
-                    precision = gr.Dropdown(
-                        choices=[i.value.name for i in Precision if i != Precision.Unknown],
-                        label="Precision",
-                        multiselect=False,
-                        value="float16",
-                        interactive=True,
-                    )
-                    weight_type = gr.Dropdown(
-                        choices=[i.value.name for i in WeightType],
-                        label="Weights type",
-                        multiselect=False,
-                        value="Original",
-                        interactive=True,
-                    )
-                    base_model_name_textbox = gr.Textbox(label="Base model (for delta or adapter weights)")
-
-            submit_button = gr.Button("Submit Eval")
+
+dimensions = ['Audience', 'Keyword', 'Format', 'Language', 'Length', 'Source']
+
+display_columns = [
+    "Rank", "Model", "WISE", "SICR", "nDCG@10(Original)", "nDCG@10(Instructed)",
+    "nDCG@10(Reversely Instructed)", "MRR@1(Original)", "MRR@1(Instructed)",
+    "MRR@1(Reversely Instructed)"
+]
+
+data_type = ["number", "markdown", "number", "number", "number", "number", "number", "number", "number", "number"]
+
+css = """
+table > thead {
+    white-space: normal
+}
+
+table {
+    --cell-width-1: 250px
+}
+
+table > tbody > tr > td:nth-child(2) > div {
+    overflow-x: auto
+}
+
+.filter-checkbox-group {
+    max-width: max-content;
+}
+
+.fixed-height-table {
+    height: 100px;
+    overflow-y: scroll;
+}
+
+"""
+
+# create Gradio interface
+with gr.Blocks() as demo:
+    gr.Markdown("# 🤗 InfoSearch Benchmark Leaderboard")
+    with gr.Tabs() as tabs:
+        with gr.TabItem("🏅 InfoSearch Benchmark"):
+            for dimension in dimensions:
+                with gr.Tab(dimension):
+                    data = get_data(f"{dimension}")
+                    gr.Dataframe(data,
+                                 headers=display_columns,
+                                 datatype=data_type,
+                                 interactive=False, elem_classes=["fixed-height-table"])
+        with gr.TabItem("🚀 Submit here!"):
+            gr.Markdown("✉️✨ Submit your model here.")
+            file_upload = gr.File(label="Upload your JSON file", file_types=["json"])
+            submit_button = gr.Button("Submit")
             submission_result = gr.Markdown()
-            submit_button.click(
-                add_new_eval,
-                [
-                    model_name_textbox,
-                    base_model_name_textbox,
-                    revision_name_textbox,
-                    precision,
-                    weight_type,
-                    model_type,
-                ],
-                submission_result,
-            )
-
-    with gr.Row():
-        with gr.Accordion("📙 Citation", open=False):
-            citation_button = gr.Textbox(
-                value=CITATION_BUTTON_TEXT,
-                label=CITATION_BUTTON_LABEL,
-                lines=20,
-                elem_id="citation-button",
-                show_copy_button=True,
-            )
+            submit_button.click(submit, file_upload, submission_result)
 
 scheduler = BackgroundScheduler()
 scheduler.add_job(restart_space, "interval", seconds=1800)

utils.py

@@ -0,0 +1,147 @@
+import os
+import random
+import json
+import pandas as pd
+
+dimensions = ['Audience', 'Keyword', 'Format', 'Language', 'Length', 'Source']
+
+
+def make_clickable_model(model_name, link):
+    return f'<a target="_blank" style="text-decoration: underline" href="{link}">{model_name}</a>'
+
+
+def rerank():
+    for dimension in dimensions:
+        with open(f"all_dimensions/{dimension}.jsonl", "r") as f:
+            data = [json.loads(line) for line in f]
+        data.sort(key=lambda x: (x["WISE"], x["SICR"]), reverse=True)
+        # 排序完后按顺序重新赋值 Rank，这个Rank是从1开始的，且放在第一列
+        for i, d in enumerate(data):
+            d["Rank"] = i + 1
+
+        with open(f"all_dimensions/{dimension}.jsonl", "w") as f:
+            for d in data:
+                # 重新构建字典，使 Rank 成为第一个键
+                ordered_d = {"Rank": d["Rank"]}
+                ordered_d.update({k: v for k, v in d.items() if k != "Rank"})
+                f.write(json.dumps(ordered_d) + "\n")
+
+
+def generate_sample_data():
+    model_names = []
+    for i in range(10):
+        model_names.append(f"Model_{i}")
+    for dimension in dimensions:
+        for model_name in model_names:
+            data = {
+                "Model": make_clickable_model(model_name, f"https://huggingface.co/"),
+                "WISE": round(random.uniform(0, 1), 2),
+                "SICR": round(random.uniform(0, 1), 2),
+                "nDCG@10(Original)": round(random.uniform(0, 1), 2),
+                "nDCG@10(Instructed)": round(random.uniform(0, 1), 2),
+                "nDCG@10(Reversely Instructed)": round(random.uniform(0, 1), 2),
+                "MRR@1(Original)": round(random.uniform(0, 1), 2),
+                "MRR@1(Instructed)": round(random.uniform(0, 1), 2),
+                "MRR@1(Reversely Instructed)": round(random.uniform(0, 1), 2),
+            }
+            with open(f"all_dimensions/{dimension}.jsonl", "a") as f:
+                f.write(json.dumps(data) + "\n")
+
+
+def get_data(dimension):
+    with open(f"all_dimensions/{dimension}.jsonl", "r") as f:
+        data = [json.loads(line) for line in f]
+
+    return pd.DataFrame(data)
+
+
+def submit(json_file):
+    flag, message = check_json_file(json_file)
+    if flag:
+        with open(json_file, "r") as f:
+            data = json.load(f)
+        if data['in_huggingface_hub']:
+            model_name = make_clickable_model(data["Model"], f"https://huggingface.co")
+        else:
+            if data["Model Link"]:
+                model_name = make_clickable_model(data["Model"], data["Model Link"])
+            else:
+                model_name = data["Model"]
+        all_dimension_data = data["dimensions"]
+        for dimension in dimensions:
+            each_dimension_data = all_dimension_data[dimension]
+            # 如果temp/{dimension}.jsonl文件不存在，则创建
+            if not os.path.exists(f"temp/{dimension}.jsonl"):
+                with open(f"temp/{dimension}.jsonl", "w"):
+                    pass
+            with open(f"temp/{dimension}.jsonl", "a") as f:
+                f.write(json.dumps({
+                    "Model": model_name,
+                    "WISE": each_dimension_data["WISE"],
+                    "SICR": each_dimension_data["SICR"],
+                    "nDCG@10(Original)": each_dimension_data["nDCG@10(Original)"],
+                    "nDCG@10(Instructed)": each_dimension_data["nDCG@10(Instructed)"],
+                    "nDCG@10(Reversely Instructed)": each_dimension_data["nDCG@10(Reversely Instructed)"],
+                    "MRR@1(Original)": each_dimension_data["MRR@1(Original)"],
+                    "MRR@1(Instructed)": each_dimension_data["MRR@1(Instructed)"],
+                    "MRR@1(Reversely Instructed)": each_dimension_data["MRR@1(Reversely Instructed)"]
+                }) + "\n")
+        return "Submission successful."
+    else:
+        return message
+
+
+def refresh():
+    if is_empty("temp"):
+        return
+    for dimension in dimensions:
+        # 读取temp/{dimension}.jsonl文件
+        with open(f"temp/{dimension}.jsonl", "r") as f:
+            data = [json.loads(line) for line in f]
+        # 将其写入all_dimensions/{dimension}.jsonl文件
+        # 如果存在相同的模型，则覆盖
+        with open(f"all_dimensions/{dimension}.jsonl", "r") as f:
+            all_data = [json.loads(line) for line in f]
+        for d in data:
+            for i, ad in enumerate(all_data):
+                if ad["Model"] == d["Model"]:
+                    all_data[i] = d
+                    break
+            else:
+                all_data.append(d)
+        with open(f"all_dimensions/{dimension}.jsonl", "w") as f:
+            for d in all_data:
+                f.write(json.dumps(d) + "\n")
+        # 删除temp/{dimension}.jsonl文件
+        os.remove(f"temp/{dimension}.jsonl")
+    rerank()
+
+
+def check_json_file(json_file):
+    with open(json_file, "r") as f:
+        try:
+            data = json.load(f)
+        except json.JSONDecodeError:
+            return False, "JSON file is not valid JSON."
+        # 检查dimensions键是否存在且是否存在对应的值（'Audience', 'Keyword', 'Format', 'Language', 'Length', 'Source'）
+        if "dimensions" not in data:
+            return False, "JSON file does not contain 'dimensions' key.",
+
+        all_dimension_data = data["dimensions"]
+        if not all([d in all_dimension_data for d in dimensions]):
+            return False, "JSON file does not contain all dimensions.",
+        # 检查每一个维度的数据是否符合要求（ WISE, SICR, nDCG@10(Original), nDCG@10(Instructed), nDCG@10(Reversely Instructed), MRR@1(Original), MRR@1(Instructed), MRR@1(Reversely Instructed)）
+
+        for d in dimensions:
+            each_dimension_data = all_dimension_data[d]
+            if not all(k in each_dimension_data for k in
+                       ["WISE", "SICR", "nDCG@10(Original)", "nDCG@10(Instructed)",
+                        "nDCG@10(Reversely Instructed)", "MRR@1(Original)", "MRR@1(Instructed)",
+                        "MRR@1(Reversely Instructed)"]):
+                return False, f"Dimension '{d}' does not contain all required keys.",
+
+    return True, "JSON file is valid."
+
+
+def is_empty(dir_path):
+    return len(os.listdir(dir_path)) == 0