evals/bible-mean-arc.csv

,gemma2-9B,gemma-7B,llama3.1-70B,llama3.1-8B,llama3-8B,llama2-7B,llama1-7B,mistral-7B,olmo-7B,avg
eng_Latn,0.6775,0.5870,0.7014,0.5794,0.5836,0.5128,0.5000,0.5862,0.4872,0.5795
fra_Latn,0.5270,0.3893,0.4208,0.3856,0.3743,0.3358,0.3041,0.3654,0.0788,0.3535
rus_Cyrl,0.5053,0.3683,0.4889,0.4067,0.4007,0.2916,0.2212,0.3413,0.0314,0.3395
nld_Latn,0.5272,0.3449,0.4362,0.4025,0.3827,0.2743,0.2358,0.2984,0.0188,0.3245
ita_Latn,0.5021,0.3588,0.4513,0.3614,0.3391,0.2919,0.2577,0.2975,0.0348,0.3216
por_Latn,0.4889,0.3622,0.4172,0.3408,0.3348,0.2829,0.2507,0.2875,0.0530,0.3131
nob_Latn,0.5103,0.3675,0.4492,0.3077,0.2909,0.2764,0.1696,0.2359,0.0199,0.2919
ukr_Cyrl,0.4650,0.3038,0.4416,0.3239,0.3195,0.2535,0.1820,0.2613,0.0036,0.2838
dan_Latn,0.5180,0.3455,0.4100,0.2883,0.2737,0.2580,0.1892,0.2444,0.0171,0.2827
spa_Latn,0.5043,0.3645,0.3832,0.3039,0.2794,0.2314,0.1799,0.2059,0.0323,0.2761
swe_Latn,0.4616,0.2828,0.4063,0.2755,0.2678,0.2482,0.2211,0.2402,0.0195,0.2692
cmn_Hani,0.4861,0.3510,0.4614,0.3515,0.3523,0.0441,0.0868,0.2180,0.0265,0.2642
bul_Cyrl,0.4694,0.2963,0.3642,0.2865,0.2830,0.1915,0.1696,0.2652,0.0092,0.2594
ind_Latn,0.4869,0.3264,0.3725,0.3459,0.3475,0.2100,0.0605,0.1457,0.0146,0.2567
pol_Latn,0.4764,0.3048,0.3582,0.2780,0.2637,0.2064,0.1728,0.2383,0.0103,0.2566
cat_Latn,0.4080,0.2830,0.3698,0.2468,0.2479,0.2271,0.2090,0.2368,0.0196,0.2498
ces_Latn,0.4224,0.2822,0.4259,0.2888,0.2752,0.1874,0.1447,0.1971,0.0062,0.2478
ron_Latn,0.4688,0.2765,0.3619,0.2780,0.2660,0.1607,0.1272,0.1631,0.0119,0.2349
afr_Latn,0.4692,0.2814,0.3900,0.2574,0.2555,0.1409,0.1142,0.1466,0.0176,0.2303
srp_Cyrl,0.3990,0.2574,0.3298,0.2238,0.2182,0.1981,0.1758,0.2194,0.0001,0.2246
deu_Latn,0.3827,0.2211,0.3522,0.2673,0.2563,0.1851,0.1359,0.1542,0.0183,0.2192
mkd_Cyrl,0.4090,0.2504,0.3671,0.2710,0.2630,0.1121,0.0933,0.1576,0.0027,0.2140
hun_Latn,0.4116,0.2166,0.3646,0.2746,0.2577,0.1294,0.0946,0.1433,0.0030,0.2106
zho_Hani,0.4114,0.2851,0.3337,0.2746,0.2795,0.0392,0.0609,0.1797,0.0178,0.2091
ell_Grek,0.4521,0.2356,0.3267,0.2986,0.3188,0.0634,0.0399,0.1390,0.0009,0.2083
slv_Latn,0.3823,0.2124,0.3196,0.1587,0.1664,0.1715,0.1377,0.1678,0.0043,0.1912
vie_Latn,0.3759,0.2134,0.3727,0.2640,0.2555,0.1162,0.0071,0.0742,0.0030,0.1869
slk_Latn,0.3947,0.2398,0.3652,0.1775,0.1626,0.1201,0.0890,0.1261,0.0053,0.1867
hrv_Latn,0.3466,0.2097,0.3244,0.1730,0.1732,0.1696,0.1372,0.1402,0.0023,0.1863
kor_Hang,0.3652,0.2044,0.3463,0.2354,0.2361,0.1103,0.0113,0.1256,0.0009,0.1817
bel_Cyrl,0.4258,0.2065,0.3281,0.2257,0.2108,0.0516,0.0310,0.0547,0.0043,0.1709
fin_Latn,0.3889,0.1912,0.3265,0.1630,0.1825,0.1471,0.0350,0.0959,0.0063,0.1707
prs_Arab,0.3580,0.2553,0.3415,0.2289,0.2110,0.0415,0.0107,0.0604,0.0026,0.1678
pes_Arab,0.3574,0.2555,0.3415,0.2291,0.2110,0.0415,0.0107,0.0604,0.0026,0.1677
arb_Arab,0.3299,0.1602,0.2759,0.1952,0.1842,0.1159,0.0662,0.1373,0.0011,0.1629
nno_Latn,0.3281,0.1942,0.3155,0.1538,0.1495,0.1049,0.0721,0.0821,0.0113,0.1568
arz_Arab,0.2860,0.1042,0.2988,0.2245,0.2254,0.0928,0.0457,0.1337,0.0000,0.1568
hin_Deva,0.4074,0.2240,0.2961,0.1894,0.1808,0.0422,0.0149,0.0483,0.0001,0.1559
tgl_Latn,0.3877,0.1784,0.3075,0.1843,0.1684,0.0537,0.0190,0.0607,0.0089,0.1521
tur_Latn,0.3052,0.2058,0.3500,0.1691,0.1600,0.0614,0.0107,0.0711,0.0001,0.1482
heb_Hebr,0.3080,0.2079,0.2146,0.1563,0.1561,0.0644,0.0321,0.0797,0.0003,0.1355
est_Latn,0.3594,0.1906,0.3106,0.1399,0.1434,0.0264,0.0059,0.0274,0.0046,0.1342
epo_Latn,0.2954,0.1448,0.3349,0.1631,0.1358,0.0344,0.0234,0.0661,0.0053,0.1337
lit_Latn,0.3943,0.1885,0.2872,0.1178,0.1121,0.0139,0.0044,0.0328,0.0003,0.1279
swh_Latn,0.3937,0.2201,0.2798,0.0869,0.1092,0.0098,0.0050,0.0121,0.0042,0.1245
isl_Latn,0.3097,0.0914,0.2414,0.1244,0.1360,0.0457,0.0174,0.0700,0.0043,0.1156
fao_Latn,0.3135,0.0898,0.2790,0.1118,0.1116,0.0380,0.0143,0.0512,0.0043,0.1126
tha_Thai,0.3159,0.0819,0.2482,0.1132,0.1271,0.0060,0.0009,0.0402,0.0001,0.1037
hye_Armn,0.3249,0.1334,0.2020,0.1043,0.1154,0.0017,0.0000,0.0057,0.0000,0.0986
ben_Beng,0.2844,0.0930,0.2350,0.1123,0.1185,0.0041,0.0004,0.0197,0.0001,0.0964
urd_Arab,0.2356,0.1165,0.2714,0.1142,0.0853,0.0121,0.0001,0.0236,0.0001,0.0954
ceb_Latn,0.2645,0.0729,0.2421,0.0885,0.0843,0.0119,0.0066,0.0257,0.0009,0.0886
npi_Deva,0.2737,0.0965,0.2156,0.0984,0.0905,0.0008,0.0001,0.0172,0.0000,0.0881
mar_Deva,0.2587,0.0886,0.2437,0.1059,0.0805,0.0009,0.0001,0.0067,0.0000,0.0872
cym_Latn,0.2316,0.0436,0.2784,0.0856,0.0888,0.0065,0.0063,0.0119,0.0006,0.0837
mlt_Latn,0.2370,0.0658,0.2272,0.0684,0.0659,0.0095,0.0066,0.0122,0.0033,0.0773
kat_Geor,0.2464,0.0943,0.1624,0.0776,0.0786,0.0021,0.0010,0.0047,0.0001,0.0741
tam_Taml,0.2665,0.1053,0.1475,0.0397,0.0443,0.0006,0.0000,0.0064,0.0000,0.0678
guj_Gujr,0.2284,0.0918,0.1653,0.0590,0.0635,0.0002,0.0000,0.0002,0.0000,0.0676
pap_Latn,0.1834,0.0570,0.2031,0.0508,0.0477,0.0211,0.0144,0.0231,0.0054,0.0673
zsm_Latn,0.1567,0.0529,0.1779,0.0719,0.0646,0.0263,0.0054,0.0245,0.0047,0.0650
mal_Mlym,0.2601,0.0920,0.1343,0.0428,0.0498,0.0005,0.0000,0.0000,0.0001,0.0644
gla_Latn,0.1399,0.0173,0.2579,0.0649,0.0639,0.0068,0.0049,0.0133,0.0047,0.0637
kaz_Cyrl,0.1284,0.0770,0.2187,0.0723,0.0683,0.0012,0.0000,0.0064,0.0000,0.0636
gle_Latn,0.1742,0.0214,0.2125,0.0634,0.0592,0.0160,0.0082,0.0103,0.0037,0.0632
tel_Telu,0.2278,0.0863,0.1395,0.0542,0.0598,0.0000,0.0000,0.0010,0.0000,0.0632
urd_Latn,0.1557,0.0531,0.1717,0.0620,0.0524,0.0133,0.0019,0.0297,0.0009,0.0601
tpi_Latn,0.2015,0.0430,0.1884,0.0348,0.0331,0.0038,0.0021,0.0147,0.0034,0.0583
asm_Beng,0.1792,0.0595,0.1771,0.0457,0.0498,0.0006,0.0000,0.0043,0.0003,0.0574
war_Latn,0.1286,0.0489,0.2045,0.0522,0.0477,0.0103,0.0038,0.0128,0.0042,0.0570
tgk_Cyrl,0.1349,0.0369,0.2222,0.0592,0.0515,0.0002,0.0000,0.0026,0.0043,0.0569
hin_Latn,0.1210,0.0647,0.1768,0.0561,0.0472,0.0148,0.0025,0.0210,0.0001,0.0560
bak_Cyrl,0.1064,0.0489,0.1923,0.0682,0.0658,0.0000,0.0000,0.0038,0.0009,0.0540
hau_Latn,0.1766,0.0605,0.1645,0.0343,0.0311,0.0035,0.0018,0.0060,0.0004,0.0532
hat_Latn,0.1292,0.0381,0.1985,0.0441,0.0345,0.0068,0.0032,0.0179,0.0042,0.0530
kir_Cyrl,0.0925,0.0535,0.2152,0.0566,0.0513,0.0002,0.0000,0.0033,0.0000,0.0525
mri_Latn,0.0887,0.0155,0.2464,0.0520,0.0424,0.0060,0.0028,0.0136,0.0001,0.0520
pan_Guru,0.2163,0.0519,0.1414,0.0252,0.0206,0.0015,0.0000,0.0000,0.0000,0.0508
san_Deva,0.1377,0.0307,0.1471,0.0508,0.0520,0.0017,0.0007,0.0221,0.0022,0.0494
kan_Knda,0.1726,0.0603,0.1257,0.0384,0.0407,0.0002,0.0000,0.0043,0.0000,0.0491
mai_Deva,0.1086,0.0434,0.1754,0.0505,0.0481,0.0018,0.0001,0.0074,0.0000,0.0484
snd_Arab,0.0853,0.0342,0.1872,0.0530,0.0434,0.0002,0.0001,0.0017,0.0000,0.0450
aeb_Arab,0.1256,0.0238,0.1345,0.0411,0.0445,0.0107,0.0079,0.0150,0.0000,0.0448
ibo_Latn,0.1126,0.0250,0.1723,0.0416,0.0362,0.0015,0.0015,0.0022,0.0006,0.0437
tuk_Latn,0.0833,0.0381,0.1797,0.0375,0.0325,0.0054,0.0021,0.0060,0.0039,0.0432
tat_Cyrl,0.0496,0.0310,0.1812,0.0556,0.0529,0.0020,0.0001,0.0022,0.0000,0.0416
zul_Latn,0.1714,0.0293,0.1127,0.0184,0.0161,0.0060,0.0032,0.0069,0.0043,0.0409
awa_Deva,0.0783,0.0320,0.1541,0.0491,0.0448,0.0018,0.0000,0.0079,0.0000,0.0409
kmr_Latn,0.0656,0.0118,0.1909,0.0401,0.0378,0.0030,0.0016,0.0060,0.0000,0.0396
ilo_Latn,0.0949,0.0275,0.1209,0.0380,0.0393,0.0086,0.0074,0.0093,0.0044,0.0389
hne_Deva,0.0692,0.0216,0.1797,0.0349,0.0295,0.0011,0.0006,0.0057,0.0001,0.0381
plt_Latn,0.1136,0.0155,0.1585,0.0160,0.0143,0.0039,0.0028,0.0069,0.0034,0.0372
sna_Latn,0.1401,0.0322,0.0986,0.0174,0.0175,0.0047,0.0024,0.0076,0.0043,0.0361
jav_Latn,0.0722,0.0220,0.1364,0.0365,0.0287,0.0072,0.0026,0.0078,0.0049,0.0354
min_Latn,0.0544,0.0356,0.1314,0.0346,0.0321,0.0100,0.0044,0.0076,0.0044,0.0349
sun_Latn,0.0654,0.0297,0.1069,0.0348,0.0325,0.0110,0.0078,0.0072,0.0056,0.0334
sot_Latn,0.0815,0.0218,0.1428,0.0140,0.0105,0.0059,0.0054,0.0064,0.0044,0.0325
sin_Sinh,0.1232,0.0271,0.0931,0.0220,0.0251,0.0000,0.0000,0.0009,0.0001,0.0324
xho_Latn,0.1090,0.0318,0.0960,0.0186,0.0129,0.0042,0.0026,0.0079,0.0044,0.0319
ary_Arab,0.0765,0.0151,0.1199,0.0263,0.0266,0.0062,0.0038,0.0088,0.0000,0.0315
uig_Arab,0.0321,0.0085,0.1626,0.0407,0.0364,0.0000,0.0001,0.0010,0.0006,0.0313
nya_Latn,0.1146,0.0210,0.0995,0.0133,0.0136,0.0053,0.0026,0.0069,0.0044,0.0312
lus_Latn,0.1457,0.0055,0.1034,0.0095,0.0089,0.0006,0.0001,0.0029,0.0034,0.0311
amh_Ethi,0.1864,0.0291,0.0600,0.0020,0.0017,0.0000,0.0000,0.0005,0.0000,0.0311
smo_Latn,0.0648,0.0100,0.1475,0.0116,0.0146,0.0039,0.0035,0.0060,0.0036,0.0295
nso_Latn,0.0540,0.0157,0.1381,0.0181,0.0120,0.0060,0.0056,0.0067,0.0043,0.0289
tsn_Latn,0.0516,0.0204,0.1284,0.0182,0.0137,0.0068,0.0047,0.0055,0.0042,0.0282
yor_Latn,0.0664,0.0063,0.1114,0.0181,0.0148,0.0017,0.0019,0.0057,0.0030,0.0255
ory_Orya,0.0757,0.0041,0.1114,0.0136,0.0136,0.0000,0.0000,0.0000,0.0001,0.0243
ssw_Latn,0.0897,0.0197,0.0737,0.0114,0.0106,0.0038,0.0032,0.0053,0.0003,0.0242
run_Latn,0.0654,0.0151,0.0759,0.0141,0.0130,0.0059,0.0060,0.0079,0.0042,0.0231
eus_Latn,0.0355,0.0187,0.0937,0.0176,0.0192,0.0071,0.0081,0.0069,0.0006,0.0230
tum_Latn,0.0494,0.0110,0.0922,0.0148,0.0137,0.0054,0.0034,0.0076,0.0039,0.0224
ckb_Arab,0.0283,0.0110,0.1162,0.0184,0.0204,0.0005,0.0000,0.0033,0.0007,0.0221
kin_Latn,0.0616,0.0138,0.0774,0.0165,0.0153,0.0039,0.0012,0.0034,0.0006,0.0215
khm_Khmr,0.0913,0.0169,0.0646,0.0078,0.0057,0.0000,0.0000,0.0016,0.0000,0.0209
som_Latn,0.0638,0.0191,0.0660,0.0133,0.0110,0.0023,0.0018,0.0045,0.0039,0.0206
bem_Latn,0.0437,0.0161,0.0687,0.0131,0.0120,0.0056,0.0047,0.0069,0.0050,0.0195
pag_Latn,0.0411,0.0163,0.0537,0.0131,0.0129,0.0101,0.0075,0.0091,0.0052,0.0188
tso_Latn,0.0289,0.0104,0.0887,0.0102,0.0093,0.0036,0.0026,0.0072,0.0044,0.0184
lug_Latn,0.0305,0.0059,0.0957,0.0087,0.0082,0.0011,0.0028,0.0028,0.0034,0.0177
mya_Mymr,0.0560,0.0134,0.0667,0.0099,0.0088,0.0000,0.0001,0.0003,0.0000,0.0172
twi_Latn,0.0245,0.0116,0.0728,0.0106,0.0103,0.0074,0.0051,0.0062,0.0039,0.0169
fij_Latn,0.0277,0.0018,0.0993,0.0055,0.0038,0.0003,0.0006,0.0009,0.0026,0.0158
ban_Latn,0.0235,0.0086,0.0592,0.0133,0.0125,0.0018,0.0004,0.0050,0.0001,0.0138
ace_Latn,0.0106,0.0108,0.0520,0.0123,0.0094,0.0041,0.0051,0.0067,0.0037,0.0128
lin_Latn,0.0157,0.0049,0.0407,0.0119,0.0118,0.0056,0.0038,0.0066,0.0027,0.0115
kik_Latn,0.0177,0.0041,0.0421,0.0068,0.0069,0.0017,0.0009,0.0064,0.0006,0.0097
quy_Latn,0.0100,0.0061,0.0425,0.0085,0.0069,0.0038,0.0029,0.0048,0.0004,0.0095
ayr_Latn,0.0086,0.0039,0.0235,0.0068,0.0058,0.0051,0.0024,0.0069,0.0033,0.0074
luo_Latn,0.0050,0.0090,0.0198,0.0072,0.0055,0.0060,0.0038,0.0047,0.0040,0.0072
wol_Latn,0.0030,0.0028,0.0359,0.0068,0.0064,0.0021,0.0021,0.0014,0.0001,0.0067
bam_Latn,0.0088,0.0053,0.0149,0.0077,0.0062,0.0047,0.0034,0.0047,0.0042,0.0066
dyu_Latn,0.0072,0.0020,0.0274,0.0039,0.0033,0.0042,0.0026,0.0043,0.0043,0.0066
bug_Latn,0.0066,0.0012,0.0221,0.0094,0.0093,0.0030,0.0037,0.0036,0.0003,0.0066
sag_Latn,0.0038,0.0051,0.0132,0.0084,0.0076,0.0050,0.0031,0.0059,0.0042,0.0062
kab_Latn,0.0044,0.0029,0.0215,0.0061,0.0046,0.0048,0.0015,0.0050,0.0037,0.0061
fuv_Latn,0.0032,0.0029,0.0163,0.0065,0.0033,0.0002,0.0022,0.0007,0.0000,0.0039
ewe_Latn,0.0028,0.0010,0.0177,0.0036,0.0039,0.0015,0.0006,0.0036,0.0004,0.0039
tir_Ethi,0.0251,0.0041,0.0041,0.0007,0.0003,0.0000,0.0000,0.0007,0.0000,0.0039
dik_Latn,0.0004,0.0035,0.0107,0.0044,0.0026,0.0026,0.0022,0.0041,0.0037,0.0038
mos_Latn,0.0016,0.0020,0.0093,0.0061,0.0050,0.0009,0.0004,0.0031,0.0014,0.0033
kac_Latn,0.0028,0.0000,0.0153,0.0046,0.0022,0.0011,0.0001,0.0010,0.0016,0.0032
kbp_Latn,0.0044,0.0018,0.0058,0.0043,0.0034,0.0002,0.0022,0.0019,0.0000,0.0027
shn_Mymr,0.0008,0.0000,0.0149,0.0015,0.0024,0.0006,0.0000,0.0000,0.0000,0.0022
taq_Latn,0.0002,0.0018,0.0111,0.0015,0.0019,0.0002,0.0000,0.0017,0.0000,0.0020
nus_Latn,0.0034,0.0000,0.0035,0.0010,0.0005,0.0023,0.0010,0.0028,0.0011,0.0017
fon_Latn,0.0018,0.0000,0.0066,0.0009,0.0009,0.0005,0.0009,0.0012,0.0001,0.0014