|
Model,NUM_Q_multich_EM,NUM_Q_multich_CC,NUM_Q_multich_PM,NUM_Q_onech_EM,NUM_Q_seq_EM,NUM_Q_seq_CC,NUM_Q_seq_PM,NUM_Q_map_EM,NUM_Q_map_CC,NUM_Q_map_PM,OPEN_Q_EM,OPEN_Q_F1,OPEN_Q_LR,LEADERBOARD |
|
claude-3.5-sonnet,84.2047930283224,87.037037037037,91.557734204793,96.0132890365448,70.6185567010309,70.6185567010309,71.1340206185567,46.7811158798283,46.7811158798283,48.9270386266094,64.9484536082474,72.1616173155497,81.745704467354,71.73300254651792 |
|
gpt-4o,73.6383442265795,78.322440087146,84.2592592592593,94.4352159468439,60.8247422680412,61.340206185567,62.8865979381443,18.0257510729614,18.4549356223176,24.2489270386266,13.4020618556701,65.2801523778733,75.1821305841924,56.17698188178635 |
|
qwen2:72b-instruct-q4_0,55.5862949563737,69.9723345392637,71.8769951053416,85.6647599237303,62.010582010582,62.1164021164021,62.1164021164021,31.4001657000828,34.3827671913836,36.8682684341342,8.92970234325522,43.0269331550944,57.1019632678911,52.38873621999514 |
|
GigaChat_Pro,63.2897603485839,66.2309368191721,75.4357298474945,76.3289036544851,52.0618556701031,52.0618556701031,52.0618556701031,11.587982832618,11.587982832618,15.8798283261803,40.893470790378,47.6443830947436,64.5051546391753,48.42843847659678 |
|
yandexgpt_pro,47.2766884531591,49.3464052287582,75.0544662309368,84.8837209302326,41.7525773195876,41.7525773195876,43.298969072165,5.5793991416309,5.5793991416309,7.29613733905579,15.4639175257732,50.7635115107509,63.6975945017182,40.90348951653745 |
|
GigaChat_Plus,51.19825708061,55.5555555555556,68.0283224400871,69.2691029900332,34.020618556701,34.020618556701,34.020618556701,6.00858369098712,6.00858369098712,8.15450643776824,32.9896907216495,40.1954759333432,58.8831615120275,38.3348535171655 |
|
GigaChat_Lite,51.19825708061,55.5555555555556,68.0283224400871,69.2691029900332,34.020618556701,34.020618556701,34.020618556701,6.00858369098712,6.00858369098712,8.15450643776824,32.6460481099656,40.3723509295672,58.9209621993127,38.32493298422898 |
|
gemma2:9b-instruct-q4_0,40.6469461587572,52.6282187699511,63.8540114918068,76.5817299358641,29.3121693121693,30.4761904761905,35.026455026455,4.30820215410108,4.88815244407622,9.48632974316487,24.4458518049398,32.0378038500143,49.7612412919569,34.88102326611132 |
|
llama3:70b-instruct-q4_0,33.4964886145989,59.8638008086827,59.1828048520962,78.1417923383602,16.6137566137566,17.8835978835979,28.4126984126984,4.14250207125104,5.2195526097763,7.24937862468931,10.3229892336922,44.8346410536552,60.4901836605447,32.75801436749228 |
|
mistral-123b,42.26579521,64.27015251,61.87363834,0.0,0.0,82.98969072,0.0,0.858369099,27.46781116,3.433476395,21.64948454,39.29104393,49.30584192,30.26 |
|
yandexgpt_lite,7.29847494553377,8.06100217864924,54.0849673202614,76.1627906976744,19.5876288659794,19.5876288659794,24.4845360824742,1.28755364806867,1.28755364806867,6.65236051502146,31.9587628865979,49.8317287269818,64.9347079037801,28.093822791159262 |
|
llama3.1:70b-instruct-q4_0,26.4311555650138,50.3298574164716,61.5450095765056,24.284971398856,8.35978835978836,15.6613756613757,26.5608465608466,1.49130074565037,2.23695111847556,10.149130074565,14.3128562381254,48.89922596918,63.82900569981,27.237805721897228 |
|
llama-405b,10.6753812636166,35.2941176470588,65.958605664488,0.913621262458472,1.54639175257732,9.27835051546392,27.5773195876289,0.858369098712446,4.29184549356223,39.6995708154506,4.46735395189003,43.3361397687549,58.5773195876289,23.267260493022395 |
|
qwen2:7b-instruct-q4_0,10.3851883379442,13.024047669717,54.7137688869972,66.2159819726122,19.5767195767196,20.5291005291005,21.2698412698413,1.90555095277548,2.31980115990058,7.49792874896438,3.73654211526282,11.4397586578317,32.7891070297657,20.41564130057174 |
|
ilyagusev/saiga_llama3,2.08555011704618,10.0021281123643,47.3824217918706,63.0438550875368,6.87830687830688,6.87830687830688,18.0952380952381,0.0,0.0,2.8169014084507,6.90310322989234,21.9392109592531,40.68397720076,17.439153827617375 |
|
phi3:14b-medium-4k-instruct-q4_0,0.0425622472866567,0.148967865503299,57.1398169823367,56.6302652106084,2.75132275132275,4.86772486772487,36.2962962962963,0.0,0.331400165700083,9.07207953603977,5.25649145028499,16.8528380209838,31.6485117162761,17.0029443931049 |
|
mistral:7b-instruct-v0.3-q4_0,0.0,0.0212811236433284,44.5520323473079,46.6458658346334,0.0,0.0,26.984126984127,0.0,0.0,3.93537696768848,2.46991766941102,11.6013570437946,26.0025332488917,12.477883939961341 |
|
solar:10.7b-instruct-v1-q4_0,0.0,0.25537348371994,48.669929772292,46.2991853007454,0.0,0.211640211640212,18.2539682539683,0.0,0.0828500414250207,3.23115161557581,3.41988600379987,11.4318388715363,23.1849271690944,11.92621159413825 |
|
random,4.04341349223239,7.59736114066823,32.698446477974,24.5103137458832,14.0740740740741,14.0740740740741,14.0740740740741,0.828500414250207,0.828500414250207,3.23115161557581,,,,11.595990952305634 |
|
wavecut/vikhr:7b-instruct_0.4-Q4_1,0.0,0.0212811236433284,36.1247073845499,34.3213728549142,0.952380952380952,2.75132275132275,6.24338624338624,0.0,0.0,0.579950289975145,10.0063331222293,19.0998530939532,34.2970234325522,11.107508557608245 |
|
llama3:8b-instruct-q4_0,0.319216854649925,1.00021281123643,47.4037029155139,26.4863927890449,0.529100529100529,1.48148148148148,4.07407407407407,0.0,0.0828500414250207,2.27837613918807,0.0633312222925902,17.1091465760483,35.2742241925269,10.469393048198624 |
|
mixtral:8x7b-instruct-v0.1-q4_0,0.0,11.8110236220472,51.0534156203448,8.66701334720055,0.105820105820106,16.9312169312169,19.4708994708995,0.0,1.65700082850041,10.4805302402651,0.633312222925902,5.19230683158126,9.84547181760608,10.449847002954446 |
|
yi:9b,4.08597573951905,11.5130878910406,34.8691210895935,25.8970358814353,8.99470899470899,10.2645502645503,12.6984126984127,0.0,0.248550124275062,1.40845070422535,0.633312222925902,3.4585674314986,15.136795440152,9.939120652487487 |
|
gemma:7b-instruct-v1.1-q4_0,2.40476697169611,12.1302404766972,30.9640349010428,27.2837580169873,4.97354497354497,11.7460317460317,5.60846560846561,0.828500414250207,0.828500414250207,2.52692626346313,0.0633312222925902,6.03356941864105,23.6782773907536,9.928457524470495 |
|
llama3.1:8b-instruct-q4_0,0.0638433709299851,0.617152585656523,46.7439880825708,8.40700294678454,0.0,1.90476190476191,6.34920634920635,0.0,0.0,4.63960231980116,0.443318556048132,19.2284319751355,39.6263457884737,9.847973375336048 |
|
qwen:7b,0.0,0.0212811236433284,30.0915088316663,37.0948171260184,8.99470899470899,10.0529100529101,11.957671957672,0.0,0.0,1.32560066280033,0.0,2.44002256870951,17.0981633945535,9.159744977898649 |
|
gemma2:27b-instruct-q4_0,1.44711640774633,6.89508406043839,36.7950627793147,22.1528861154446,2.53968253968254,8.57142857142857,8.57142857142857,0.0,0.0828500414250207,0.787075393537697,1.64661177960735,8.23779148465489,15.1519949335022,8.683000975246989 |
|
yi:6b,0.595871462013194,8.93807193019791,21.5896999361566,12.619171433524,0.317460317460317,9.73544973544974,1.74603174603175,0.165700082850041,0.828500414250207,0.414250207125104,0.189993666877771,2.29393179599174,10.265357821406,5.361499273025722 |
|
llama2:13b,0.0,0.0425622472866567,28.2400510746967,0.0,0.0,0.105820105820106,2.75132275132275,0.0,0.0,0.124275062137531,0.0633312222925902,5.16796089780207,13.0633312222926,3.812204198742385 |
|
AnatoliiPotapov/T-lite-instruct-0.1,0.0,7.29847494553376,52.1786492374727,0.0,0.0,17.0103092783505,17.0103092783505,0.0,0.858369098712446,2.57510729613733,0.0,4.52000213091168,9.50859106529209,8.535370179289309 |
|
Среднее значение,20.422671397760762,27.460168212211308,53.39834082949528,44.60746395894965,16.713903889161614,22.297459953762353,24.435189730722378,4.735537689666839,6.078146908927527,9.437473109299502,12.136533079873367,27.02488259909779,40.83412427928588,23.698010251848952 |
|
|