update leaderboard 2024-09-11

data_v2/bosc_zh_mc_gen.csv

@@ -14,9 +14,9 @@ Mistral-7B,0.0,0.0,37.5,37.5,20.0,20.0,50.0,50.0
 Qwen-14B-Chat,47.5,45.0,50.0,47.5,50.0,47.5,55.0,57.5
 Qwen-72B-Chat,50.0,50.0,47.5,47.5,45.0,45.0,60.0,60.0
 Yi-34B-Chat,55.0,55.0,60.0,67.5,50.0,50.0,52.5,55.0
-Claude-3-Opus,,72.85714285714286,,,,,,
+Claude-3-Opus,72.85714285714286,72.85714285714286,,,,,,
 Gemma_2B,37.5,37.5,40.0,40.0,32.5,32.5,40.0,40.0
 Gemma_7B,32.5,32.5,62.5,62.5,40.0,40.0,50.0,50.0
-Meta-Llama-3-8B-Instruct,,52.85714285714286,,47.14285714285714,,52.85714285714286,,30.0
+Meta-Llama-3-8B-Instruct,52.85714285714286,52.85714285714286,47.14285714285714,47.14285714285714,52.85714285714286,52.85714285714286,30.0,30.0
 Qwen1.5-14B-Base,47.5,47.5,50.0,50.0,47.5,47.5,45.0,45.0
 Qwen1.5-14B-Chat,45.0,47.5,60.0,72.5,52.5,55.0,60.0,60.0

data_v2/gtja_zh_mc_gen.csv

@@ -14,9 +14,9 @@ Mistral-7B,5.49,5.49,47.25,47.25,14.29,14.29,38.46,38.46
 Qwen-14B-Chat,47.25,47.25,53.85,54.95,54.95,54.95,59.34,61.54
 Qwen-72B-Chat,71.43,71.43,67.03,67.03,70.33,70.33,74.73,74.73
 Yi-34B-Chat,71.43,74.73,71.43,73.63,69.23,70.33,49.45,47.25
-Claude-3-Opus,,41.508438818565395,,,,,,
+Claude-3-Opus,41.508438818565395,41.508438818565395,,,,,,
 Gemma_2B,30.76923,30.76923,43.95604,43.95604,32.96703,32.96703,29.67033,29.67033
 Gemma_7B,29.67033,29.67033,56.04396,56.04396,34.06593,34.06593,50.54945,50.54945
-Meta-Llama-3-8B-Instruct,,36.550632911392405,,38.08016877637131,,43.24894514767932,,34.28270042194093
+Meta-Llama-3-8B-Instruct,36.550632911392405,36.550632911392405,38.08016877637131,38.08016877637131,43.24894514767932,43.24894514767932,34.28270042194093,34.28270042194093
 Qwen1.5-14B-Base,53.84615,53.84615,63.73626,63.73626,68.13187,68.13187,42.85714,42.85714
 Qwen1.5-14B-Chat,56.04396,54.94505,67.03297,68.13187,59.34066,57.14286,60.43956,62.63736

data_v2/huaweicloud_zh_mc_gen.csv

@@ -14,9 +14,9 @@ Mistral-7B,0.0,0.0,23.33,23.33,0.0,0.0,16.67,16.67
 Qwen-14B-Chat,13.33,13.33,20.0,26.67,40.0,30.0,26.67,33.33
 Qwen-72B-Chat,36.67,36.67,33.33,33.33,43.33,43.33,33.33,36.67
 Yi-34B-Chat,36.67,40.0,36.67,30.0,50.0,46.67,30.0,43.33
-Claude-3-Opus,,55.0,,,,,,
+Claude-3-Opus,55.0,55.0,,,,,,
 Gemma_2B,26.66667,26.66667,10.0,10.0,26.66667,26.66667,20.0,20.0
 Gemma_7B,3.333333,3.333333,23.33333,23.33333,13.33333,13.33333,30.0,30.0
-Meta-Llama-3-8B-Instruct,,27.5,,22.5,,30.0,,30.0
+Meta-Llama-3-8B-Instruct,27.5,27.5,22.5,22.5,30.0,30.0,30.0,30.0
 Qwen1.5-14B-Base,20.0,20.0,33.33333,33.33333,20.0,20.0,30.0,30.0
 Qwen1.5-14B-Chat,23.33333,26.66667,13.33333,13.33333,26.66667,26.66667,20.0,30.0

data_v2/inspur_en_mc_gen.csv

@@ -1,8 +1,8 @@
-name,zero_self_con,zero_cot_self_con,few_self_con,few_cot_self_con
-GPT-4,85.71428571428571,87.75510204081633,90.47619047619048,91.15646258503402
-GPT-4o,89.79591836734694,90.47619047619048,91.15646258503402,92.51700680272108
-Baichuan2-7B-Chat,44.89795918367347,66.66666666666666,28.57142857142857,50.34013605442177
-Claude-3-Opus,87.75510204081633,89.1156462585034,91.15646258503402,88.43537414965986
-Qwen2-0.5B-Instruct,,53.06122448979592,,52.38095238095239
-Qwen2-1.5B-Instruct,,67.3469387755102,65.98639455782312,
-Qwen2-7B-Instruct,80.95238095238095,,80.27210884353741,82.31292517006803
+name,zero_naive,zero_self_con,zero_cot,zero_cot_self_con,few_naive,few_self_con,few_cot,few_cot_self_con
+GPT-4,85.71428571428571,85.71428571428571,87.75510204081633,87.75510204081633,90.47619047619048,90.47619047619048,91.15646258503402,91.15646258503402
+GPT-4o,89.79591836734694,89.79591836734694,90.47619047619048,90.47619047619048,91.15646258503402,91.15646258503402,92.51700680272108,92.51700680272108
+Baichuan2-7B-Chat,44.89795918367347,44.89795918367347,66.66666666666666,66.66666666666666,28.57142857142857,28.57142857142857,50.34013605442177,50.34013605442177
+Claude-3-Opus,87.75510204081633,87.75510204081633,89.1156462585034,89.1156462585034,91.15646258503402,91.15646258503402,88.43537414965986,88.43537414965986
+Qwen2-0.5B-Instruct,,,53.06122448979592,53.06122448979592,,,52.38095238095239,52.38095238095239
+Qwen2-1.5B-Instruct,,,67.3469387755102,67.3469387755102,65.98639455782312,65.98639455782312,,
+Qwen2-7B-Instruct,80.95238095238095,80.95238095238095,,,80.27210884353741,80.27210884353741,82.31292517006803,82.31292517006803

data_v2/inspur_zh_mc_gen.csv

@@ -1,8 +1,8 @@
-name,zero_self_con,zero_cot_self_con,few_self_con,few_cot_self_con
-GPT-4,,87.07482993197279,87.07482993197279,91.15646258503402
-GPT-4o,87.07482993197279,89.1156462585034,89.1156462585034,91.15646258503402
-Baichuan2-7B-Chat,62.585034013605444,,42.857142857142854,
-Claude-3-Opus,83.6734693877551,85.03401360544217,87.75510204081633,91.83673469387756
-Qwen2-0.5B-Instruct,56.4625850340136,,,57.14285714285714
-Qwen2-1.5B-Instruct,,68.02721088435374,,
-Qwen2-7B-Instruct,76.19047619047619,80.95238095238095,76.87074829931973,
+name,zero_naive,zero_self_con,zero_cot,zero_cot_self_con,few_naive,few_self_con,few_cot,few_cot_self_con
+GPT-4,,,87.07482993197279,87.07482993197279,87.07482993197279,87.07482993197279,91.15646258503402,91.15646258503402
+GPT-4o,87.07482993197279,87.07482993197279,89.1156462585034,89.1156462585034,89.1156462585034,89.1156462585034,91.15646258503402,91.15646258503402
+Baichuan2-7B-Chat,62.585034013605444,62.585034013605444,,,42.857142857142854,42.857142857142854,,
+Claude-3-Opus,83.6734693877551,83.6734693877551,85.03401360544217,85.03401360544217,87.75510204081633,87.75510204081633,91.83673469387756,91.83673469387756
+Qwen2-0.5B-Instruct,56.4625850340136,56.4625850340136,,,,,57.14285714285714,57.14285714285714
+Qwen2-1.5B-Instruct,,,68.02721088435374,68.02721088435374,,,,
+Qwen2-7B-Instruct,76.19047619047619,76.19047619047619,80.95238095238095,80.95238095238095,76.87074829931973,76.87074829931973,,

data_v2/lenovo_zh_mc_gen.csv

@@ -12,5 +12,5 @@ Mistral-7B,47.5,47.5,62.5,62.5,35.0,35.0,60.0,60.0
 Qwen-14B-Chat,70.0,67.5,70.0,67.5,70.0,65.0,65.0,67.5
 Qwen-72B-Chat,72.5,72.5,75.0,75.0,75.0,75.0,75.0,75.0
 Yi-34B-Chat,75.0,75.0,87.5,82.5,62.5,57.5,52.5,52.5
-Claude-3-Opus,,71.42857142857143,,,,,,
-Meta-Llama-3-8B-Instruct,,47.14285714285714,,44.285714285714285,,45.714285714285715,,32.857142857142854
+Claude-3-Opus,71.42857142857143,71.42857142857143,,,,,,
+Meta-Llama-3-8B-Instruct,47.14285714285714,47.14285714285714,44.285714285714285,44.285714285714285,45.714285714285715,45.714285714285715,32.857142857142854,32.857142857142854

data_v2/network_en_mc_gen.csv

@@ -21,9 +21,9 @@ Qwen-14B-Chat,43.78,47.81,56.58,59.4,62.09,59.7,49.06,55.88
 Qwen-72B-Chat,70.41,70.5,72.38,72.56,70.32,70.32,70.13,70.22
 Qwen-7B-Chat,45.9,46.0,47.3,50.1,52.1,51.0,48.3,49.8
 Yi-34B-Chat,57.75,59.14,65.11,68.79,68.16,68.37,78.09,80.06
-Claude-3-Opus,,69.03417341637355,,,,,,
+Claude-3-Opus,69.03417341637355,69.03417341637355,,,,,,
 Gemma_2B,26.46048,26.46048,33.41924,33.41924,26.6323,26.6323,37.54296,37.54296
 Gemma_7B,25.08591,25.08591,50.85911,50.85911,30.24055,30.24055,51.55747,51.55747
-Meta-Llama-3-8B-Instruct,,38.279481659390655,,76.69172932330827,,23.734458771084668,,33.241749376506874
+Meta-Llama-3-8B-Instruct,38.279481659390655,38.279481659390655,76.69172932330827,76.69172932330827,23.734458771084668,23.734458771084668,33.241749376506874,33.241749376506874
 Qwen1.5-14B-Base,34.87973,34.87973,60.82474,60.82474,65.54983,65.54983,47.07904,47.07904
 Qwen1.5-14B-Chat,54.89691,56.4433,64.08935,67.09622,52.23368,53.52234,59.53608,64.17526

data_v2/network_zh_mc_gen.csv

@@ -24,9 +24,9 @@ Qwen-14B-Chat,48.35,48.81,55.35,57.4,58.53,56.12,52.12,54.99
 Qwen-72B-Chat,65.77,65.86,68.13,68.3,69.4,69.4,69.99,70.08
 Qwen-7B-Chat,29.6,29.9,50.6,53.5,50.4,46.9,46.9,47.7
 Yi-34B-Chat,61.61,62.56,68.11,69.75,65.73,65.37,69.88,71.21
-Claude-3-Opus,,62.329525111479995,,,,,,
+Claude-3-Opus,62.329525111479995,62.329525111479995,,,,,,
 Gemma_2B,29.69019,29.69019,39.15663,39.15663,29.77625,29.77625,38.64028,38.64028
 Gemma_7B,31.58348,31.58348,47.59036,47.59036,34.68158,34.68158,48.88124,48.88124
-Meta-Llama-3-8B-Instruct,,35.904696806952444,,38.94801939914722,,41.717931191615406,,31.059792337987826
+Meta-Llama-3-8B-Instruct,35.904696806952444,35.904696806952444,38.94801939914722,38.94801939914722,41.717931191615406,41.717931191615406,31.059792337987826,31.059792337987826
 Qwen1.5-14B-Base,45.18072,45.18072,59.1222,59.1222,61.10155,61.10155,52.4957,52.4957
 Qwen1.5-14B-Chat,54.04475,53.87263,62.56454,63.85542,58.77797,58.0895,63.42513,65.57659

data_v2/oracle_en_mc_gen.csv

@@ -21,9 +21,9 @@ Qwen-14B-Chat,24.95,28.37,33.0,36.62,27.97,28.37,27.97,24.14
 Qwen-72B-Chat,47.28,47.48,48.09,48.09,49.7,49.7,43.46,43.66
 Qwen-7B-Chat,18.91,19.11,22.13,23.94,26.76,25.55,34.81,33.4
 Yi-34B-Chat,47.08,48.69,47.08,46.28,58.15,58.35,56.94,58.95
-Claude-3-Opus,,48.31816996021653,,,,,,
+Claude-3-Opus,48.31816996021653,48.31816996021653,,,,,,
 Gemma_2B,16.90141,16.90141,19.5171,19.5171,16.09658,16.09658,24.74849,24.74849
 Gemma_7B,14.28571,14.28571,30.98592,30.98592,2.60223,2.60223,43.85965,43.85965
-Meta-Llama-3-8B-Instruct,,28.468825409248026,,40.47805387073632,,23.33528989760647,,34.6197743429205
+Meta-Llama-3-8B-Instruct,28.468825409248026,28.468825409248026,40.47805387073632,40.47805387073632,23.33528989760647,23.33528989760647,34.6197743429205,34.6197743429205
 Qwen1.5-14B-Base,29.17505,29.17505,33.60161,33.60161,36.82093,36.82093,27.7666,27.7666
 Qwen1.5-14B-Chat,32.79678,35.41247,39.43662,43.05835,32.39437,33.60161,36.82093,38.833

data_v2/oracle_zh_mc_gen.csv

@@ -20,10 +20,10 @@ Qwen-14B-Chat,27.57,27.57,32.39,36.02,40.04,35.41,30.38,33.4
 Qwen-72B-Chat,48.29,48.49,49.5,49.7,49.7,49.7,45.27,44.87
 Qwen-7B-Chat,18.51,17.71,27.36,28.37,29.78,29.58,33.6,31.79
 Yi-34B-Chat,49.9,49.3,52.72,53.72,56.34,56.34,51.31,54.33
-Claude-3-Opus,,50.00570664579664,,,,,,
+Claude-3-Opus,50.00570664579664,50.00570664579664,,,,,,
 Gemma_2B,18.51107,18.51107,24.9497,24.9497,21.52918,21.52918,27.7666,27.7666
 Gemma_7B,19.3159,19.3159,53.94737,53.94737,18.51107,18.51107,5.204461,5.204461
-Hisense,,42.6270136307311,,43.228657144720536,,37.94267266679711,,45.94991195460771
-Meta-Llama-3-8B-Instruct,,33.91785690993282,,27.773429857170807,,41.359323028761494,,32.62733972477663
+Hisense,42.6270136307311,42.6270136307311,43.228657144720536,43.228657144720536,37.94267266679711,37.94267266679711,45.94991195460771,45.94991195460771
+Meta-Llama-3-8B-Instruct,33.91785690993282,33.91785690993282,27.773429857170807,27.773429857170807,41.359323028761494,41.359323028761494,32.62733972477663,32.62733972477663
 Qwen1.5-14B-Base,20.92555,20.92555,35.61368,35.61368,41.44869,41.44869,30.78471,30.78471
 Qwen1.5-14B-Chat,24.14487,23.34004,40.64386,41.04628,38.22938,38.02817,39.43662,40.04024

data_v2/pufa_zh_mc_gen.csv

@@ -14,10 +14,10 @@ Mistral-7B,4.0,4.0,58.67,58.67,22.67,22.67,54.67,54.67
 Qwen-14B-Chat,73.33,73.33,69.33,72.0,73.33,73.33,72.0,80.0
 Qwen-72B-Chat,90.67,90.67,85.33,85.33,88.0,88.0,82.67,82.67
 Yi-34B-Chat,84.0,84.0,88.0,88.0,90.67,92.0,78.67,89.33
-Claude-3-Opus,,93.24324324324324,,,,,,
+Claude-3-Opus,93.24324324324324,93.24324324324324,,,,,,
 Gemma_2B,36.0,36.0,41.33333,41.33333,36.0,36.0,30.66667,30.66667
 Gemma_7B,34.66667,34.66667,56.0,56.0,46.66667,46.66667,56.0,56.0
-Hisense,,92.56756756756756,,93.91891891891892,,43.91891891891892,,93.91891891891892
-Meta-Llama-3-8B-Instruct,,85.8108108108108,,31.756756756756754,,83.1081081081081,,27.7027027027027
+Hisense,92.56756756756756,92.56756756756756,93.91891891891892,93.91891891891892,43.91891891891892,43.91891891891892,93.91891891891892,93.91891891891892
+Meta-Llama-3-8B-Instruct,85.8108108108108,85.8108108108108,31.756756756756754,31.756756756756754,83.1081081081081,83.1081081081081,27.7027027027027,27.7027027027027
 Qwen1.5-14B-Base,78.66667,78.66667,72.0,72.0,92.0,92.0,42.66667,42.66667
 Qwen1.5-14B-Chat,86.66667,89.33333,85.33333,85.33333,78.66667,80.0,86.66667,85.33333

data_v2/rzy_zh_mc_gen.csv

@@ -14,9 +14,9 @@ Mistral-7B,18.53,18.53,60.3,60.3,29.88,29.88,59.75,59.75
 Qwen-14B-Chat,61.96,61.55,61.55,64.45,65.28,63.49,62.93,65.98
 Qwen-72B-Chat,66.67,66.67,65.28,65.28,65.98,65.98,70.12,70.12
 Yi-34B-Chat,64.45,64.59,67.77,67.36,60.17,60.03,57.68,57.54
-Claude-3-Opus,,67.64288271089369,,,,,,
+Claude-3-Opus,67.64288271089369,67.64288271089369,,,,,,
 Gemma_2B,36.37621,36.37621,45.22822,45.22822,33.60996,33.60996,37.75934,37.75934
 Gemma_7B,39.41909,39.41909,54.77178,54.77178,42.04703,42.04703,56.70816,56.70816
-Meta-Llama-3-8B-Instruct,,36.55172413793103,,28.27586206896552,,38.62068965517241,,34.48275862068966
+Meta-Llama-3-8B-Instruct,36.55172413793103,36.55172413793103,28.27586206896552,28.27586206896552,38.62068965517241,38.62068965517241,34.48275862068966,34.48275862068966
 Qwen1.5-14B-Base,51.17566,51.17566,62.6556,62.6556,65.42185,65.42185,50.89903,50.89903
 Qwen1.5-14B-Chat,62.93223,64.03873,64.59198,64.31535,63.34716,63.7621,65.42185,65.9751

data_v2/zabbix_zh_mc_gen.csv

@@ -14,9 +14,9 @@ Mistral-7B,6.0,6.0,42.0,42.0,11.0,11.0,44.0,44.0
 Qwen-14B-Chat,36.0,36.0,39.0,41.0,44.0,40.0,47.0,43.0
 Qwen-72B-Chat,46.0,46.0,44.0,44.0,45.0,45.0,61.0,61.0
 Yi-34B-Chat,40.0,40.0,40.0,40.0,42.0,42.0,42.0,42.0
-Claude-3-Opus,,61.71875,,,,,,
+Claude-3-Opus,61.71875,61.71875,,,,,,
 Gemma_2B,25.0,25.0,32.0,32.0,24.0,24.0,30.0,30.0
 Gemma_7B,22.0,22.0,44.0,44.0,28.0,28.0,40.0,40.0
-Meta-Llama-3-8B-Instruct,,39.670138888888886,,37.58680555555556,,30.381944444444443,,33.072916666666664
+Meta-Llama-3-8B-Instruct,39.670138888888886,39.670138888888886,37.58680555555556,37.58680555555556,30.381944444444443,30.381944444444443,33.072916666666664,33.072916666666664
 Qwen1.5-14B-Base,38.0,38.0,39.0,39.0,48.0,48.0,36.0,36.0
 Qwen1.5-14B-Chat,34.0,34.0,45.0,43.0,42.0,39.0,48.0,49.0

data_v2/zjyd_zh_mc_gen.csv

@@ -21,9 +21,9 @@ Qwen-14B-Chat,41.71,41.44,45.58,47.98,53.52,49.92,54.72,58.85
 Qwen-72B-Chat,64.79,64.79,65.79,65.72,70.19,70.19,68.31,68.38
 Qwen-7B-Chat,36.28,36.5,33.18,33.51,41.58,40.59,31.48,31.46
 Yi-34B-Chat,64.91,64.58,62.77,65.51,70.85,70.92,48.77,47.97
-Claude-3-Opus,,68.05555555555556,,,,,,
+Claude-3-Opus,68.05555555555556,68.05555555555556,,,,,,
 Gemma_2B,25.6,25.6,28.3,28.3,19.1,19.1,35.5,35.5
 Gemma_7B,27.3,27.3,35.4,35.4,17.3,17.3,44.5,44.5
-Meta-Llama-3-8B-Instruct,,63.425925925925924,,0.0,,66.2037037037037,,25.0
+Meta-Llama-3-8B-Instruct,63.425925925925924,63.425925925925924,0.0,0.0,66.2037037037037,66.2037037037037,25.0,25.0
 Qwen1.5-14B-Base,49.1,49.1,49.9,49.9,62.5,62.5,41.3,41.3
 Qwen1.5-14B-Chat,38.6,38.9,48.8,50.5,54.6,55.2,52.1,52.7

data_v2/zte_en_mc_gen.csv

@@ -19,10 +19,10 @@ Qwen-14B-Chat,33.71,36.25,41.24,42.51,51.19,50.39,57.18,59.18
 Qwen-72B-Chat,53.19,53.19,55.25,55.52,58.13,58.13,58.72,58.99
 Qwen-7B-Chat,33.37,33.74,32.97,34.1,32.98,32.7,36.6,36.65
 Yi-34B-Chat,38.24,37.04,48.24,52.1,61.33,61.19,53.53,53.39
-Claude-3-Opus,,49.599999999999994,,,,,,
+Claude-3-Opus,49.599999999999994,49.599999999999994,,,,,,
 Gemma_2B,20.1,20.1,24.2,24.2,31.2,31.2,35.5,35.5
 Gemma_7B,23.1,23.1,34.4,34.4,21.4,21.4,33.1,33.1
-Meta-Llama-3-70B-Instruct,,38.9,,63.4,,37.6,,59.0
-Meta-Llama-3-8B-Instruct,,24.7,,35.4,,19.7,,32.9
+Meta-Llama-3-70B-Instruct,38.9,38.9,63.4,63.4,37.6,37.6,59.0,59.0
+Meta-Llama-3-8B-Instruct,24.7,24.7,35.4,35.4,19.7,19.7,32.9,32.9
 Qwen1.5-14B-Base,34.0,34.0,42.8,42.8,57.9,57.9,40.2,40.2
 Qwen1.5-14B-Chat,34.5,35.6,41.7,41.1,33.2,34.7,46.2,47.4

data_v2/zte_zh_mc_gen.csv

@@ -21,10 +21,10 @@ Qwen-14B-Chat,41.71,41.44,45.58,47.98,53.52,49.92,54.72,58.85
 Qwen-72B-Chat,64.79,64.79,65.79,65.72,70.19,70.19,68.31,68.38
 Qwen-7B-Chat,36.28,36.5,33.18,33.51,41.58,40.59,31.48,31.46
 Yi-34B-Chat,64.91,64.58,62.77,65.51,70.85,70.92,48.77,47.97
-Claude-3-Opus,,51.4,,,,,,
+Claude-3-Opus,51.4,51.4,,,,,,
 Gemma_2B,25.6,25.6,28.3,28.3,19.1,19.1,35.5,35.5
 Gemma_7B,27.3,27.3,35.4,35.4,17.3,17.3,44.5,44.5
-Meta-Llama-3-70B-Instruct,,31.1,,37.4,,51.10000000000001,,36.900000000000006
-Meta-Llama-3-8B-Instruct,,31.1,,34.3,,36.0,,37.1
+Meta-Llama-3-70B-Instruct,31.1,31.1,37.4,37.4,51.10000000000001,51.10000000000001,36.900000000000006,36.900000000000006
+Meta-Llama-3-8B-Instruct,31.1,31.1,34.3,34.3,36.0,36.0,37.1,37.1
 Qwen1.5-14B-Base,49.1,49.1,49.9,49.9,62.5,62.5,41.3,41.3
 Qwen1.5-14B-Chat,38.6,38.9,48.8,50.5,54.6,55.2,52.1,52.7