add openllm-v2

README.md

@@ -162,11 +162,44 @@ This version of the lm-evaluation-harness includes versions of MMLU, ARC-Challen
   <tr>
    <td><strong>Arena Hard</strong>
    </td>
-   <td>85.0
+   <td><strong>85.0</strong>
    </td>
-   <td>84.5
+   <td><strong>84.5</strong>
    </td>
-   <td>99.41%
+   <td><strong>99.41%</strong>
+   </td>
+  </tr>
+  <tr>
+   <td><strong>OpenLLM Leaderboard v1</strong>
+   </td>
+   <td><strong>80.13</strong>
+   </td>
+   <td><strong>80.29</strong>
+   </td>
+   <td><strong>100.2%</strong>
+   </td>
+  </tr>
+  <tr>
+   <td><strong>OpenLLM Leaderboard v2</strong>
+   </td>
+   <td><strong>40.25</strong>
+   </td>
+   <td><strong>39.82</strong>
+   </td>
+   <td><strong>98.93%</strong>
+   </td>
+  </tr>
+</table>
+
+<table>
+  <tr>
+   <td><strong>Benchmark (per-task breakdown)</strong>
+   </td>
+   <td><strong>nvidia/Llama-3.1-Nemotron-70B-Instruct-HF</strong>
+   </td>
+   <td><strong>neuralmagic/Llama-3.1-Nemotron-70B-Instruct-HF-FP8-dynamic (this model)</strong>
+   </td>
+   <td><strong>Recovery</strong>
    </td>
   </tr>
   <tr>
@@ -243,7 +276,6 @@ This version of the lm-evaluation-harness includes versions of MMLU, ARC-Challen
    <td>102.05%
    </td>
   </tr>
-  <tr>
    <td><strong>Average</strong>
    </td>
    <td><strong>80.13</strong>
@@ -252,7 +284,6 @@ This version of the lm-evaluation-harness includes versions of MMLU, ARC-Challen
    </td>
    <td><strong>100.2%</strong>
    </td>
-  </tr>
   <tr>
    <td><strong>OpenLLM v2</strong>
    </td>
@@ -260,11 +291,11 @@ This version of the lm-evaluation-harness includes versions of MMLU, ARC-Challen
   <tr>
    <td>MMLU-Pro (5-shot)
    </td>
-   <td>ToDo
+   <td>43.45
    </td>
-   <td>ToDo
+   <td>42.99
    </td>
-   <td>ToDo
+   <td>98.94%
    </td>
   </tr>
   <tr>
@@ -280,11 +311,11 @@ This version of the lm-evaluation-harness includes versions of MMLU, ARC-Challen
   <tr>
    <td>BBH (3-shot)
    </td>
-   <td>ToDo
+   <td>47.12
    </td>
-   <td>ToDo
+   <td>46.88
    </td>
-   <td>ToDo
+   <td>99.5%
    </td>
   </tr>
   <tr>
@@ -297,16 +328,6 @@ This version of the lm-evaluation-harness includes versions of MMLU, ARC-Challen
    <td>91.32%
    </td>
   </tr>
-  <tr>
-   <td>GPQA (0-shot)
-   </td>
-   <td>34.05
-   </td>
-   <td>35.97
-   </td>
-   <td>105.63%
-   </td>
-  </tr>
   <tr>
    <td>MuSR (0-shot)
    </td>
@@ -320,11 +341,11 @@ This version of the lm-evaluation-harness includes versions of MMLU, ARC-Challen
   <tr>
    <td><strong>Average</strong>
    </td>
-   <td><strong>ToDo</strong>
+   <td><strong>40.25</strong>
    </td>
-   <td><strong>ToDo</strong>
+   <td><strong>39.82</strong>
    </td>
-   <td><strong>ToDo</strong>
+   <td><strong>98.93%</strong>
    </td>
   </tr>
 </table>