File size: 4,774 Bytes
e9065c7 5eb492b e9065c7 5eb492b e9065c7 5eb492b e9065c7 5eb492b e9065c7 5eb492b | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 | import csv
# ---------------------------------------------------------------------------
# IOL-AI 2024 - organizer demonstration submission.
#
# This script is NOT a model. It embeds a curated answer key and deliberately
# produces THREE kinds of predictions so you can watch chrF, exact_match and the
# geometric-mean score DIVERGE on the leaderboard:
#
# EXACT -> gold answer verbatim -> exact_match = 1, chrF = 1
# NEAR -> gold answer with a tiny typo -> exact_match = 0, chrF high (partial)
# blank -> id not in the key -> exact_match = 0, chrF = 0
#
# Because the NEAR bucket scores 0 on exact_match but high on chrF, you'll see
# chrF > score > exact_match
# instead of the three collapsing to one number (which happens when every item
# is all-or-nothing). Use it only to smoke-test the leaderboard end to end.
#
# !!! KEEP THE MODEL REPO YOU UPLOAD THIS TO PRIVATE !!!
# It embeds gold answers; a public repo would leak them.
# ---------------------------------------------------------------------------
# Map of {sub-question id: correct answer} for a ~52% subset of points.
ANSWERS = {
"012024010102": "you(du) will bite me",
"012024010201": "jelhuŋnet",
"012024010204": "nekunŋivŋətək",
"012024020102": "C",
"012024020201": "car (= short lorry)",
"012024020303": "ruubiitcha puphubii",
"012024020304": "mu’akoeta uhuyitibee",
"012024020305": "makuitcha eratibii",
"012024030101": "Kurai",
"012024030102": "Trafe",
"012024030106": "Nfiyam",
"012024030109": "Tawth",
"012024030201": "bäiŋam rä",
"012024010101": "you(sg) lead him",
"012024010103": "I caught them(pl)",
"012024010104": "I will wait for you(pl)",
"012024010105": "we(pl) send him",
"012024010202": "mətəjgolan",
"012024010203": "kenakmellaŋtək",
"012024010205": "inelletək",
"012024020101": "D",
"012024020103": "B",
"012024020104": "A",
"012024020202": "tall cooking pots",
"012024020203": "female thief",
"012024020204": "zebras",
"012024020205": "(short, thick) tail",
"012024020206": "leopards",
"012024020301": "uphukwama gogogogo",
"012024020302": "shumukosa dongoko",
"012024020306": "wiribiisa pophoko",
"012024030103": "Mea",
"012024030104": "Naimr",
"012024030105": "Skri",
"012024030107": "Marua",
"012024030108": "Wafine",
"012024030110": "Abia",
"012024030111": "Wims",
"012024030112": "Gwam",
"012024030113": "Nakre",
"012024030114": "Maraga",
"012024030115": "Mabata",
"012024030202": "enat yé",
"012024030204": "nge yé"
}
# Ids whose prediction is a deliberate NEAR-MISS (right idea, one-character typo).
# These score 0 on exact_match but high on chrF, so the two metrics diverge.
# Chosen to be long enough that a single typo still leaves high character overlap.
NEAR_MISS_IDS = {
"012024010102", # "you(du) will bite me"
"012024010104", # "I will wait for you(pl)"
"012024020201", # "car (= short lorry)"
"012024020303", # "ruubiitcha puphubii"
"012024020304", # "mu’akoeta uhuyitibee"
"012024020305", # "makuitcha eratibii"
"012024020202", # "tall cooking pots"
"012024020301", # "uphukwama gogogogo"
"012024020302", # "shumukosa dongoko"
"012024020306", # "wiribiisa pophoko"
"012024010103", # "I caught them(pl)"
"012024010202", # "mətəjgolan"
}
def near_miss(s):
"""Return a near-miss copy of `s`: transpose the first interior pair of
differing letters. Guarantees exact_match = 0 while keeping nearly every
character (and most n-grams) intact, so chrF stays high."""
chars = list(s)
for i in range(1, len(chars) - 1):
a, b = chars[i], chars[i + 1]
if a != b and a.isalnum() and b.isalnum():
chars[i], chars[i + 1] = b, a
return "".join(chars)
# fallback: duplicate the last character
return s + s[-1] if s else s
TEST = "/tmp/data/test.csv" # competition test set, mounted by the platform
rows = []
n_exact = n_near = n_blank = 0
with open(TEST, newline="") as f:
for r in csv.DictReader(f):
rid = str(r["id"]).strip()
if rid in NEAR_MISS_IDS:
pred = near_miss(ANSWERS[rid])
n_near += 1
elif rid in ANSWERS:
pred = ANSWERS[rid]
n_exact += 1
else:
pred = ""
n_blank += 1
rows.append({"id": r["id"], "pred": pred})
with open("submission.csv", "w", newline="") as f:
w = csv.DictWriter(f, fieldnames=["id", "pred"])
w.writeheader()
w.writerows(rows)
print(f"Wrote submission.csv with {len(rows)} rows; "
f"{n_exact} exact, {n_near} near-miss, {n_blank} blank.")
|