Upload from GitHub Actions: Use task subset for average score

evals/backend.py

@@ -28,6 +28,8 @@ task_metrics = [
     "mgsm_accuracy",
 ]
 
+task_metrics_basic = ["translation_from_bleu", "translation_to_bleu", "classification_accuracy"]
+
 
 def compute_normalized_average(df, metrics):
     """Compute average of min-max normalized metric columns."""
@@ -52,7 +54,7 @@ def make_model_table(df, models):
     df["task_metric"] = df["task"] + "_" + df["metric"]
     df = df.drop(columns=["task", "metric"])
     df = df.pivot(index="model", columns="task_metric", values="score")
-    df["average"] = compute_normalized_average(df, task_metrics)
+    df["average"] = compute_normalized_average(df, task_metrics_basic)
     df = df.sort_values(by="average", ascending=False).reset_index()
     df = pd.merge(df, models, left_on="model", right_on="id", how="left")
     df["rank"] = df.index + 1
@@ -84,7 +86,7 @@ def make_language_table(df, languages):
     df["task_metric"] = df["task"] + "_" + df["metric"]
     df = df.drop(columns=["task", "metric"])
     df = df.pivot(index="bcp_47", columns="task_metric", values="score").reset_index()
-    df["average"] = compute_normalized_average(df, task_metrics)
+    df["average"] = compute_normalized_average(df, task_metrics_basic)
     df = pd.merge(languages, df, on="bcp_47", how="outer")
     df = df.sort_values(by="speakers", ascending=False)
     df = df[

frontend/src/components/LanguagePlot.js

@@ -3,38 +3,38 @@ import * as Plot from '@observablehq/plot'
 
 const LanguagePlot = ({ data, width = 750, height = 500 }) => {
   const containerRef = useRef()
-  const languages = data.language_table.filter(a => a.translation_from_bleu > 0)
+  const languages = data.language_table.filter(a => a.average > 0)
   const families = [...new Set(languages.map(a => a.family))]
 
   useEffect(() => {
     const plot = Plot.plot({
       width: width,
       height: height,
-      subtitle: 'Translation quality by language',
+      subtitle: 'Proficiency scores by language',
       x: {
         label: 'Number of Speakers',
         type: 'log'
       },
       y: {
-        label: 'Translation quality (spBLEU score for translating from the given language to other languages)'
+        label: 'Language proficiency score'
       },
       marks: [
         Plot.dot(languages, {
           x: 'speakers',
-          y: d => d.translation_from_bleu,
+          y: d => d.average,
           r: 'speakers',
           fill: 'family',
           title: d =>
             `${d.language_name}\n${d.speakers.toLocaleString('en-US', {
               notation: 'compact'
-            })} speakers\nScore: ${d.translation_from_bleu.toFixed(2)}`,
+            })} speakers\nScore: ${d.average.toFixed(2)}`,
           tip: true
         }),
         Plot.text(
           languages.filter(a => a.speakers > 1e8),
           {
             x: 'speakers',
-            y: d => d.translation_from_bleu,
+            y: d => d.average,
             text: d => d.language_name,
             fill: 'black',
             frameAnchor: 'left',

frontend/src/components/ScoreColumns.js

@@ -13,8 +13,8 @@ const scoreBodyTemplate = (field, options = {}) => {
 const ScoreColumns = [
   <Column
     field='average'
-    header='Overall'
-    headerTooltip='Language Proficiency Score (average of all displayed scores, after min-max normalization)'
+    header='Proficiency'
+    headerTooltip='Language Proficiency Score (average translation and classification scores, after min-max normalization)'
     sortable
     body={scoreBodyTemplate('average', { minScore: 0.2, maxScore: 0.5 })}
     style={{ minWidth: '5rem', maxWidth: '10rem' }}