script.py

import csv

# ---------------------------------------------------------------------------
# IOL-AI 2024 - organizer demonstration submission.
#
# This script is NOT a model. It embeds a curated answer key and deliberately
# produces THREE kinds of predictions so you can watch chrF, exact_match and the
# geometric-mean score DIVERGE on the leaderboard:
#
#   EXACT  -> gold answer verbatim        -> exact_match = 1, chrF = 1
#   NEAR   -> gold answer with a tiny typo -> exact_match = 0, chrF high (partial)
#   blank  -> id not in the key            -> exact_match = 0, chrF = 0
#
# Because the NEAR bucket scores 0 on exact_match but high on chrF, you'll see
#   chrF  >  score  >  exact_match
# instead of the three collapsing to one number (which happens when every item
# is all-or-nothing). Use it only to smoke-test the leaderboard end to end.
#
# !!! KEEP THE MODEL REPO YOU UPLOAD THIS TO PRIVATE !!!
# It embeds gold answers; a public repo would leak them.
# ---------------------------------------------------------------------------

# Map of {sub-question id: correct answer} for a ~52% subset of points.
ANSWERS = {
    "012024010102": "you(du) will bite me",
    "012024010201": "jelhuŋnet",
    "012024010204": "nekunŋivŋətək",
    "012024020102": "C",
    "012024020201": "car (= short lorry)",
    "012024020303": "ruubiitcha puphubii",
    "012024020304": "mu’akoeta uhuyitibee",
    "012024020305": "makuitcha eratibii",
    "012024030101": "Kurai",
    "012024030102": "Trafe",
    "012024030106": "Nfiyam",
    "012024030109": "Tawth",
    "012024030201": "bäiŋam rä",
    "012024010101": "you(sg) lead him",
    "012024010103": "I caught them(pl)",
    "012024010104": "I will wait for you(pl)",
    "012024010105": "we(pl) send him",
    "012024010202": "mətəjgolan",
    "012024010203": "kenakmellaŋtək",
    "012024010205": "inelletək",
    "012024020101": "D",
    "012024020103": "B",
    "012024020104": "A",
    "012024020202": "tall cooking pots",
    "012024020203": "female thief",
    "012024020204": "zebras",
    "012024020205": "(short, thick) tail",
    "012024020206": "leopards",
    "012024020301": "uphukwama gogogogo",
    "012024020302": "shumukosa dongoko",
    "012024020306": "wiribiisa pophoko",
    "012024030103": "Mea",
    "012024030104": "Naimr",
    "012024030105": "Skri",
    "012024030107": "Marua",
    "012024030108": "Wafine",
    "012024030110": "Abia",
    "012024030111": "Wims",
    "012024030112": "Gwam",
    "012024030113": "Nakre",
    "012024030114": "Maraga",
    "012024030115": "Mabata",
    "012024030202": "enat yé",
    "012024030204": "nge yé"
}

# Ids whose prediction is a deliberate NEAR-MISS (right idea, one-character typo).
# These score 0 on exact_match but high on chrF, so the two metrics diverge.
# Chosen to be long enough that a single typo still leaves high character overlap.
NEAR_MISS_IDS = {
    "012024010102",   # "you(du) will bite me"
    "012024010104",   # "I will wait for you(pl)"
    "012024020201",   # "car (= short lorry)"
    "012024020303",   # "ruubiitcha puphubii"
    "012024020304",   # "mu’akoeta uhuyitibee"
    "012024020305",   # "makuitcha eratibii"
    "012024020202",   # "tall cooking pots"
    "012024020301",   # "uphukwama gogogogo"
    "012024020302",   # "shumukosa dongoko"
    "012024020306",   # "wiribiisa pophoko"
    "012024010103",   # "I caught them(pl)"
    "012024010202",   # "mətəjgolan"
}


def near_miss(s):
    """Return a near-miss copy of `s`: transpose the first interior pair of
    differing letters. Guarantees exact_match = 0 while keeping nearly every
    character (and most n-grams) intact, so chrF stays high."""
    chars = list(s)
    for i in range(1, len(chars) - 1):
        a, b = chars[i], chars[i + 1]
        if a != b and a.isalnum() and b.isalnum():
            chars[i], chars[i + 1] = b, a
            return "".join(chars)
    # fallback: duplicate the last character
    return s + s[-1] if s else s


TEST = "/tmp/data/test.csv"   # competition test set, mounted by the platform

rows = []
n_exact = n_near = n_blank = 0
with open(TEST, newline="") as f:
    for r in csv.DictReader(f):
        rid = str(r["id"]).strip()
        if rid in NEAR_MISS_IDS:
            pred = near_miss(ANSWERS[rid])
            n_near += 1
        elif rid in ANSWERS:
            pred = ANSWERS[rid]
            n_exact += 1
        else:
            pred = ""
            n_blank += 1
        rows.append({"id": r["id"], "pred": pred})

with open("submission.csv", "w", newline="") as f:
    w = csv.DictWriter(f, fieldnames=["id", "pred"])
    w.writeheader()
    w.writerows(rows)

print(f"Wrote submission.csv with {len(rows)} rows; "
      f"{n_exact} exact, {n_near} near-miss, {n_blank} blank.")