Upload from GitHub Actions: Evaluate on autotranslated GSM dataset

datasets.json

@@ -300,7 +300,22 @@
         "parallel": true,
         "translation": "machine",
         "base": "MGSM",
-        "implemented": true,
+        "implemented": false,
+        "group": "Grade School Math"
+    },
+    {
+        "name": "GSM Auto-Translated",
+        "author": null,
+        "author_url": null,
+        "url": null,
+        "n_languages": 52,
+        "tasks": [
+            "math"
+        ],
+        "parallel": true,
+        "translation": "machine",
+        "base": "MGSM",
+        "implemented": false,
         "group": "Grade School Math"
     },
     {

evals/datasets_/mgsm.py

@@ -1,5 +1,12 @@
+import asyncio
+import os
+
+from datasets import Dataset, load_dataset
 from datasets_.util import _get_dataset_config_names, _load_dataset
-from langcodes import Language, standardize_tag
+from langcodes import standardize_tag
+from models import google_supported_languages, translate_google
+from tqdm import tqdm
+from tqdm.asyncio import tqdm_asyncio
 
 slug_mgsm = "juletxara/mgsm"
 tags_mgsm = {
@@ -14,6 +21,12 @@ tags_gsm8kx = {
     standardize_tag(a, macro=True): a
     for a in _get_dataset_config_names(slug_gsm8kx, trust_remote_code=True)
 }
+slug_gsm_autotranslated = "fair-forward/gsm-autotranslated"
+tags_gsm_autotranslated = {
+    standardize_tag(a, macro=True): a
+    for a in _get_dataset_config_names(slug_gsm_autotranslated)
+}
+
 
 def parse_number(i):
     if isinstance(i, int):
@@ -23,6 +36,7 @@ def parse_number(i):
     except ValueError:
         return None
 
+
 def load_mgsm(language_bcp_47, nr):
     if language_bcp_47 in tags_mgsm.keys():
         ds = _load_dataset(slug_mgsm, subset=tags_mgsm[language_bcp_47], split="test")
@@ -32,6 +46,11 @@ def load_mgsm(language_bcp_47, nr):
             slug_afrimgsm, subset=tags_afrimgsm[language_bcp_47], split="test"
         )
         return slug_afrimgsm, ds[nr]
+    elif language_bcp_47 in tags_gsm_autotranslated.keys():
+        ds = _load_dataset(
+            slug_gsm_autotranslated, subset=tags_gsm_autotranslated[language_bcp_47], split="test"
+        )
+        return slug_gsm_autotranslated, ds[nr]
     elif language_bcp_47 in tags_gsm8kx.keys():
         row = _load_dataset(
             slug_gsm8kx,
@@ -43,3 +62,39 @@ def load_mgsm(language_bcp_47, nr):
         return slug_gsm8kx, row
     else:
         return None, None
+
+
+def translate_mgsm(languages):
+    human_translated = [*tags_mgsm.keys(), *tags_afrimgsm.keys()]
+    untranslated = [
+        lang
+        for lang in languages["bcp_47"].values[:100]
+        if lang not in human_translated and lang in google_supported_languages
+    ]
+    en = _load_dataset(slug_mgsm, subset=tags_mgsm["en"], split="test")
+    slug = "fair-forward/gsm-autotranslated"
+    for lang in tqdm(untranslated):
+        # check if already exists on hub
+        try:
+            ds_lang = load_dataset(slug, lang, split="test")
+        except ValueError:
+            print(f"Translating {lang}...")
+            questions_tr = [translate_google(q, "en", lang) for q in en["question"]]
+            questions_tr = asyncio.run(tqdm_asyncio.gather(*questions_tr))
+            ds_lang = Dataset.from_dict(
+                {
+                    "question": questions_tr,
+                    "answer": en["answer"],
+                    "answer_number": en["answer_number"],
+                    "equation_solution": en["equation_solution"],
+                }
+            )
+            ds_lang.push_to_hub(
+                slug,
+                split="test",
+                config_name=lang,
+                token=os.getenv("HUGGINGFACE_ACCESS_TOKEN"),
+            )
+            ds_lang.to_json(
+                f"data/mgsm/{lang}.json", lines=False, force_ascii=False, indent=2
+            )

evals/main.py

@@ -1,5 +1,4 @@
 import asyncio
-from time import time
 
 import pandas as pd
 from languages import languages
@@ -16,7 +15,7 @@ n_sentences = 10
 
 async def evaluate():
     # FIXME we should not need this for-loop, but it helps
-    for n_languages in range(100, 101):
+    for n_languages in range(90, 101, 3):
         print(f"running evaluations for {n_languages} languages")
         old_results = pd.read_json("results.json")
         old_models = pd.read_json("models.json")

evals/models.py

@@ -34,7 +34,7 @@ important_models = [
     "mistralai/mistral-small-3.1-24b-instruct",  # 0.3$
     "mistralai/mistral-saba",  # 0.6$
     "mistralai/mistral-nemo",  # 0.08$
-    "google/gemini-2.5-flash-preview",  # 0.6$
+    "google/gemini-2.5-flash",  # 0.6$
     "google/gemini-2.0-flash-lite-001",  # 0.3$
     "google/gemma-3-27b-it",  # 0.2$
     # "qwen/qwen-turbo", # 0.2$; recognizes "inappropriate content"
@@ -50,8 +50,14 @@ important_models = [
 
 blocklist = [
     "microsoft/wizardlm-2-8x22b",  # temporarily rate-limited
-    "google/gemini-2.5-pro",  # something wrong FIXME
-    "google/gemini-2.5-pro-preview",  # something wrong FIXME
+    "google/gemini-2.5-pro-preview",
+    "google/gemini-2.5-flash-preview",
+    "google/gemini-2.5-flash-lite-preview",
+    "google/gemini-2.5-flash-preview-04-17",
+    "google/gemini-2.5-flash-preview-05-20",
+    "google/gemini-2.5-flash-lite-preview-06-17",
+    "google/gemini-2.5-pro-preview-06-05",
+    "google/gemini-2.5-pro-preview-05-06",
 ]
 
 transcription_models = [
@@ -153,17 +159,11 @@ async def complete(**kwargs) -> str | None:
 
 
 translate_client = translate.Client()
-supported_languages = [l["language"] for l in translate_client.get_languages()]
+google_supported_languages = [l["language"] for l in translate_client.get_languages()]
 
 
 @cache
 async def translate_google(text, source_language, target_language):
-    source_language = closest_supported_match(source_language, supported_languages)
-    target_language = closest_supported_match(target_language, supported_languages)
-    if source_language == target_language:
-        return text
-    if source_language is None or target_language is None:
-        return None
     async with google_rate_limit:
         response = translate_client.translate(
             text, source_language=source_language, target_language=target_language
@@ -284,7 +284,7 @@ def load_models(date: date):
         ["translation_from", "translation_to", "classification", "mmlu", "mgsm"]
     ] * len(models)
     models = pd.concat([models, get_translation_models()])
-    models = models[ # temporary fix FIXME
+    models = models[  # temporary fix FIXME
         (models["id"] != "google/gemini-2.5-pro")
         & (models["id"] != "google/gemini-2.5-pro-preview")
     ]

evals/tasks.py

@@ -1,12 +1,15 @@
 import random
 from functools import partial
 from textwrap import dedent
+
 import evaluate
 import pandas as pd
 import sentencepiece as spm
 from datasets_.flores import flores_sentences
 from datasets_.mgsm import load_mgsm, parse_number
 from datasets_.mmlu import load_mmlu
+from google.cloud import translate_v2 as translate
+from langcodes import closest_supported_match
 from languages import languages, script_name
 from models import complete, transcribe, translate_google
 
@@ -22,6 +25,9 @@ target_languages = languages[languages["in_benchmark"]].sample(
     frac=1, weights="speakers", replace=True, random_state=42
 )
 
+translate_client = translate.Client()
+supported_languages = [l["language"] for l in translate_client.get_languages()]
+
 
 async def translate_and_evaluate(model, bcp_47, sentence_nr, mode="from"):
     original_language = languages[languages["bcp_47"] == bcp_47].iloc[0]
@@ -40,9 +46,18 @@ async def translate_and_evaluate(model, bcp_47, sentence_nr, mode="from"):
     target_sentence = flores_sentences(target_language)["text"][sentence_nr].strip()
     script = script_name(target_language.flores_path.split("_")[1])
     if model == "google/translate-v2":
-        prediction = await translate_google(
-            original_sentence, original_language.bcp_47, target_language.bcp_47
+        original_language = closest_supported_match(
+            original_language, supported_languages
         )
+        target_language = closest_supported_match(target_language, supported_languages)
+        if original_language == target_language:
+            prediction = original_sentence
+        elif original_language is None or target_language is None:
+            prediction = None
+        else:
+            prediction = await translate_google(
+                original_sentence, original_language.bcp_47, target_language.bcp_47
+            )
     else:
         prediction = await complete(
             model=model,

evals/translate.py

@@ -0,0 +1,5 @@
+from languages import languages
+from datasets_.mgsm import translate_mgsm
+
+if __name__ == "__main__":
+    translate_mgsm(languages)